Merge 18d3f42 into 2b1caad

.travis.yml

@@ -6,6 +6,9 @@ cache:
 
 sudo: false
 
+services:
+  - postgresql
+
 python:
 - '2.7'
 - '3.5'
@@ -34,10 +37,10 @@ install:
 - pip install pytest
 - pip install pytest-coverage
 - pip install coveralls
-- python setup.py install
+- pip install .[all,dev]
 
 script:
-- pytest python/sdssdb/tests --cov python/sdssdb --cov-report html
+- pytest -p no:sugar python/sdssdb/tests --cov python/sdssdb --cov-report html
 
 after_success:
 - coveralls

python/sdssdb/connection.py

@@ -59,18 +59,25 @@ class DatabaseConnection(six.with_metaclass(abc.ABCMeta)):
     autoconnect : bool
         Whether to autoconnect to the database using the profile parameters.
         Requites `.dbname` to be set.
+    dbversion : str
+        A database version.  If specified, appends to dbname as "dbname_dbversion"
+        and becomes the dbname used for connection strings.
 
     """
 
     #: The database name.
     dbname = None
+    dbversion = None
 
-    def __init__(self, dbname=None, profile=None, autoconnect=True):
+    def __init__(self, dbname=None, profile=None, autoconnect=True, dbversion=None):
 
         #: Reports whether the connection is active.
         self.connected = False
         self.profile = None
         self.dbname = dbname if dbname else self.dbname
+        self.dbversion = dbversion or self.dbversion
+        if self.dbversion:
+            self.dbname = f'{self.dbname}_{self.dbversion}' 
 
         self.set_profile(profile=profile, connect=autoconnect)
 
@@ -303,6 +310,18 @@ def become_user(self):
 
         self.become(user)
 
+    def change_version(self, dbversion=None):
+        ''' Change database version and attempt to reconnect
+        
+        Parameters:
+            dbversion (str):
+                A database version
+        '''
+        self.dbversion = dbversion
+        dbname, *dbver = self.dbname.split('_')
+        self.dbname = f'{dbname}_{self.dbversion}' if dbversion else dbname
+        self.connect(dbname=self.dbname, silent_on_fail=True)
+
 
 if _peewee:
 
@@ -426,6 +445,8 @@ def _conn(self, dbname, silent_on_fail=False, **params):
                 self.engine.dispose()
                 self.engine = None
                 self.connected = False
+                self.Session = None
+                self.metadata = None
             else:
                 self.connected = True
                 self.dbname = dbname
@@ -436,6 +457,7 @@ def _conn(self, dbname, silent_on_fail=False, **params):
         def reset_engine(self):
             ''' Reset the engine, metadata, and session '''
 
+            self.bases = []
             if self.engine:
                 self.engine.dispose()
                 self.engine = None

python/sdssdb/etc/sdssdb.yml

@@ -51,3 +51,9 @@ utahdb:
     host: db.sdss.utah.edu
     port: 5432
     domain: db.sdss.utah.edu
+
+slore:
+    user: sdss
+    host: lore.sdss.utah.edu
+    port: 5432
+    domain: lore.sdss.utah.edu

python/sdssdb/sqlalchemy/__init__.py

@@ -99,11 +99,11 @@ def cone_search(self, ra, dec, a, b=None, pa=None, ra_col='ra', dec_col='dec'):
         dec_attr = getattr(self, dec_col)
 
         if b is None:
-            return fn.q3c_radial_query(ra_attr, dec_attr, ra, dec, a)
+            return func.q3c_radial_query(ra_attr, dec_attr, ra, dec, a)
         else:
             pa = pa or 0.0
             ratio = b / a
-            return fn.q3c_ellipse_query(ra_attr, dec_attr, ra, dec, a, ratio, pa)
+            return func.q3c_ellipse_query(ra_attr, dec_attr, ra, dec, a, ratio, pa)
 
     @cone_search.expression
     def cone_search(cls, ra, dec, a, b=None, pa=None, ra_col='ra', dec_col='dec'):

python/sdssdb/sqlalchemy/archive/__init__.py

@@ -19,8 +19,9 @@
 
 
 class ArchiveDatabaseConnection(SQLADatabaseConnection):
-    dbname = 'archive_20190507'
+    dbname = 'archive'
+    dbversion = '20200302'
     base = ArchiveBase
 
 
-database = ArchiveDatabaseConnection(autoconnect=True)
+database = ArchiveDatabaseConnection(autoconnect=True, profile='local')

python/sdssdb/sqlalchemy/archive/sas.py

@@ -13,6 +13,8 @@
 from sdssdb.sqlalchemy.archive import database, ArchiveBase
 from sqlalchemy.ext.declarative import AbstractConcreteBase, declared_attr
 from sqlalchemy.orm import relationship
+from sqlalchemy.ext.hybrid import hybrid_property
+from sqlalchemy import func
 
 SCHEMA = 'sas'
 
@@ -50,10 +52,14 @@ class File(Base):
     __tablename__ = 'file'
     print_fields = ['name']
 
-    @property
+    @hybrid_property
     def name(self):
         return self.location.rsplit('/', 1)[-1]
 
+    @name.expression
+    def name(cls):
+        return func.reverse(func.split_part(func.reverse(cls.location), '/', 1))
+
 
 class SymlinkFile(Base):
     __tablename__ = 'symlink_file'

python/sdssdb/sqlalchemy/sdss5db/catalogdb.py

@@ -193,6 +193,9 @@ class TwoMassCleanNoNeighbor(Base):
 class DR14QV44(Base):
     __tablename__ = 'dr14q_v4_4'
 
+    sdss_name = Column(String, primary_key=True)
+
+
 
 def define_relations():
 

python/sdssdb/tests/conftest.py

@@ -3,17 +3,102 @@
 # conftest.py
 #
 
+from __future__ import print_function, division, absolute_import, unicode_literals
 
-from __future__ import division
-from __future__ import print_function
-from __future__ import absolute_import
-from __future__ import unicode_literals
+import re
+import pytest
+import importlib
+import inspect
+from sdssdb.tests.sqladbs import prepare_testdb as sqla_prepdb
+from sdssdb.tests.pwdbs import prepare_testdb as pw_prepdb
+from pytest_postgresql.factories import DatabaseJanitor
 
 
-"""
-Here you can add fixtures that will be used for all the tests in this
-directory. You can also add conftest.py files in underlying subdirectories.
-Those conftest.py will only be applies to the tests in that subdirectory and
-underlying directories. See https://docs.pytest.org/en/2.7.3/plugins.html for
-more information.
-"""
+def pytest_addoption(parser):
+    """ Add new options to the pytest command-line """
+    # only run peewee tests
+    parser.addoption('--peewee', action='store_true', default=False, 
+                     help='Only run tests for peewee dbs')
+
+    # only run sqla tests
+    parser.addoption('--sqla', action='store_true', default=False,
+                     help='Only run tests for sqlalchemy dbs')
+
+    # persist the sqla session and peewee transaction
+    parser.addoption('--persist-sessions', action='store_true', default=False,
+                     help='Switch session and transaction fixtures to module scope')
+
+
+def pytest_ignore_collect(path, config):
+    ''' pytest hook to identify tests to be ignored during collection
+    
+    Looks through all test_xxx.py files in pwdbs and sqladbs and determines
+    which ones have databases that fail to connect and ignores them.
+
+    '''
+    only_peewee = config.getoption("--peewee", None)
+    only_sqla = config.getoption("--sqla", None)
+    assert not all([only_peewee, only_sqla]), 'both --peewee and --sqla options cannot be set'
+    if only_peewee:
+        if 'sqladbs' in str(path):
+            return True
+    if only_sqla:
+        if 'pwdbs' in str(path):
+            return True
+
+    # identify and ignore test modules where no local
+    # database is set up for those tests
+    if re.search('test_[a-z]+.py', str(path)):
+        # get module name
+        modname = inspect.getmodulename(path)
+        # find and load the underlying module
+        spec = importlib.util.spec_from_file_location(modname, path)
+        foo = importlib.util.module_from_spec(spec)
+        # execute load
+        spec.loader.exec_module(foo)
+        # get the database from the module
+        db = getattr(foo, 'database', None)
+        # check if db is connected
+        if db and db.connected is False and db.dbname != 'test':
+            return True
+
+
+@pytest.fixture(scope='module', autouse=True)
+def skipdb(database):
+    ''' fixture to skip database tests if the db does not exist '''
+    if database.connected is False:
+        pytest.skip(f'no {database.dbname} found')
+        database = None
+
+
+@pytest.fixture(scope='module')
+def dropdb():
+    janitor = DatabaseJanitor('postgres', 'localhost', 5432, 'test', '11.4')
+    janitor.drop()
+
+
+@pytest.fixture(scope='module')
+def database(dropdb, request):
+    ''' Module fixture to initialize a real database or a test postgresql database '''
+    if hasattr(request, 'param'):
+        # yield a real database
+        yield request.param
+    else:
+        # check if request is coming from a sqla db or peewee db
+        issqla = 'sqladbs' in request.module.__name__ or 'sqlalchemy' in request.module.__name__
+        # initialize the test database
+        # uses https://github.com/ClearcodeHQ/pytest-postgresql
+        janitor = DatabaseJanitor('postgres', 'localhost', 5432, 'test', '11.4')
+        janitor.init()
+        db = sqla_prepdb() if issqla else pw_prepdb()
+        yield db
+        db = None
+        janitor.drop()
+
+
+def determine_scope(fixture_name, config):
+    ''' determine the scope of the session and transaction fixtures '''
+    if config.getoption("--persist-sessions", None):
+        return "module"
+    return "function"
+

python/sdssdb/tests/pwdbs/__init__.py

@@ -0,0 +1,40 @@
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+# 
+# Filename: __init__.py
+# Project: peewee
+# Author: Brian Cherinka
+# Created: Sunday, 1st March 2020 1:47:12 pm
+# License: BSD 3-clause "New" or "Revised" License
+# Copyright (c) 2020 Brian Cherinka
+# Last Modified: Monday, 23rd March 2020 5:03:45 pm
+# Modified By: Brian Cherinka
+
+
+from __future__ import print_function, division, absolute_import
+
+from sdssdb.connection import PeeweeDatabaseConnection
+from sdssdb.peewee import BaseModel
+
+
+class TmpDatabaseConnection(PeeweeDatabaseConnection):
+    dbname = 'test'
+
+
+database = TmpDatabaseConnection(autoconnect=False, profile='local')
+
+
+# Create a new base temp model class
+class TmpModel(BaseModel):
+
+    class Meta:
+        database = database
+
+
+def prepare_testdb():
+    ''' connect and set up test models after db initialization ''' 
+    models = TmpModel.__subclasses__()
+    database.bind(models, bind_refs=False, bind_backrefs=False)
+    database.connect()
+    database.create_tables(models)
+    return database

python/sdssdb/tests/pwdbs/conftest.py

@@ -0,0 +1,43 @@
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+# 
+# Filename: conftest.py
+# Project: pwdbs
+# Author: Brian Cherinka
+# Created: Monday, 23rd March 2020 4:41:32 pm
+# License: BSD 3-clause "New" or "Revised" License
+# Copyright (c) 2020 Brian Cherinka
+# Last Modified: Tuesday, 24th March 2020 2:53:13 pm
+# Modified By: Brian Cherinka
+
+
+from __future__ import print_function, division, absolute_import
+import factory
+import inspect
+import pytest
+from pytest_factoryboy import register
+from . import factories
+from ..conftest import determine_scope
+
+
+# register all xxxFactory classes
+#
+# pytest_factoryboy @register decorator registers fixtures in directory where it is
+# called.  To make available to tests, need to either import them into conftest or register
+# manually in conftest.  The below code registers them manually instead of with
+# @register class decorator. For docs on pytest_factoryboy,
+# see https://pytest-factoryboy.readthedocs.io/en/latest
+for item in dir(factories):
+    if item == 'PeeweeModelFactory':
+        continue
+    attr = getattr(factories, item)
+    if inspect.isclass(attr) and issubclass(attr, factory.Factory):
+        register(attr)
+
+
+@pytest.fixture(scope=determine_scope, autouse=True)
+def transaction(database):
+    ''' Peewee transaction fixture. set autouse=True to ensure persistence '''
+    with database.transaction() as txn:
+        yield txn
+        txn.rollback()