Skip to content
This repository has been archived by the owner on Apr 9, 2021. It is now read-only.

Commit

Permalink
StationLite exclusively uses localconfig URL info
Browse files Browse the repository at this point in the history 
commit 7f03094a9c8d1bf486f21747bd43bb82b3bb38e1
Author: Daniel Armbruster <daniel.armbruster@sed.ethz.ch>
Date:   Tue May 1 00:29:59 2018 +0200

    exemplary DB

commit 23cd1e55285a182c3aa4ff9afcf1d0835340231d
Author: Daniel Armbruster <daniel.armbruster@sed.ethz.ch>
Date:   Tue May 1 00:18:08 2018 +0200

    new stationlite.db.empty due to DB model changes

commit 202f5a5c7b2c238a200a190a3ebde3967cbe141d
Author: Daniel Armbruster <daniel.armbruster@sed.ethz.ch>
Date:   Mon Apr 30 23:34:35 2018 +0200

    fetch fdsnws-station URL from localconfig

    fdsnws-station URL is used to resolve the route's streamepoch
    information.

commit 0c1de2682285152e469f4501fd14305cb4e313f1
Author: Daniel Armbruster <daniel.armbruster@sed.ethz.ch>
Date:   Mon Apr 30 17:50:36 2018 +0200

    populate services/endpoints from localconfigs

commit a8a37be4326b1fee0571fed29107c7a90676411c
Author: Daniel Armbruster <daniel.armbruster@sed.ethz.ch>
Date:   Mon Apr 30 15:35:24 2018 +0200

    remove db.init()

    Do not prefill the DB with EIDA_NODE information from `settings.py`.
    Simply create the schemas.

commit 8c17f15281f597de1521818ceb33a9ebffde4757
Author: Daniel Armbruster <daniel.armbruster@sed.ethz.ch>
Date:   Mon Apr 30 15:26:51 2018 +0200

    remove node specific information @ StationLite DB

commit 136d3b67f64e27dd47f1c89c3de8abe966487e4e
Author: Daniel Armbruster <daniel.armbruster@sed.ethz.ch>
Date:   Mon Apr 30 15:21:53 2018 +0200

    use obspy.UTCDateTime to parse ISO8601 formatted strings

    While FDSN webservices do not allow timezone information, however,
    eidaws-routing does within localconfig configuration files.

commit e9d62ca470e39af0ee4cc03e9e1dcd0d883da990
Author: Daniel Armbruster <daniel.armbruster@sed.ethz.ch>
Date:   Mon Apr 30 15:21:10 2018 +0200

    adjust BGR WFCatalog URL

commit bc7b8427d55f7412a7218159c9d8f25d5bc44db3
Author: Daniel Armbruster <daniel.armbruster@sed.ethz.ch>
Date:   Mon Apr 30 15:20:51 2018 +0200

    correction
  • Loading branch information
Daniel Armbruster committed Apr 30, 2018
1 parent 1d0017e commit c0fd56f
Show file tree
Hide file tree
Showing 7 changed files with 124 additions and 256 deletions.
Binary file modified db/stationlite.db.empty
View file
Binary file not shown.
4 changes: 2 additions & 2 deletions eidangservices/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,9 +206,9 @@
'uri_path_config_vnet': '/eidaws/routing/1/localconfig',
'static_file': ''},
'wfcatalog': {
'url': 'http://eida.bgr.de/eidaws/wfcatalog/alpha/query',
'url': 'http://eida.bgr.de/eidaws/wfcatalog/1/query',
'server': 'http://eida.bgr.de',
'uri_path_query': "/eidaws/wfcatalog/alpha/query"}
'uri_path_query': "/eidaws/wfcatalog/1/query"}
}
},
'testquerysncls': {
Expand Down
118 changes: 3 additions & 115 deletions eidangservices/stationlite/engine/db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,31 +32,14 @@

from builtins import * # noqa

import collections
import logging

from contextlib import contextmanager

from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound


from eidangservices import settings
from eidangservices.stationlite.engine import orm
from eidangservices.utils.error import Error, ErrorWithTraceback
from eidangservices.utils.sncl import (none_as_max, max_as_none,
StreamEpochs, StreamEpochsHandler)


# TODO(damb): Find a more elegant solution for CACHED_SERVICES workaround.
CACHED_SERVICES = ('station', 'dataselect', 'wfcatalog')

CACHED_SERVICES_FDSN = ('station', 'dataselect')
CACHED_SERVICES_EIDA = ('wfcatalog',)
CACHED_SERVICES = {
'fdsn': CACHED_SERVICES_FDSN,
'eida': CACHED_SERVICES_EIDA
}


logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -128,102 +111,8 @@ def session_guard(session):
finally:
session.close()

# session_guard ()

def get_cached_services():
retval = []
for k, v in CACHED_SERVICES.items():
retval.extend(v)
return retval


def init(session):
"""initialize database tables"""

def _lookup_service(services, name, std):
return next(
(x for x in services if x.name == name and x.standard == std),
None)

# _lookup_service ()

# populate db tables from the mediatorws configuration (i.e. currently
# settings.py)
logger.debug('Collecting mappings ...')
_services = []
for service_std, services in CACHED_SERVICES.items():
for s_name in services:
_services.append(orm.Service(name=s_name,
standard=service_std))

_nodes = []
_endpoints = []

logger.debug('Services available: %s' % _services)

for node_name, node_par in settings.EIDA_NODES.items():

try:
n = orm.Node(name=node_name,
description=node_par['name'])

# add services to node
# NOTE(damb): Only such services are added which are both
# cached and have configured parameters in settings.py

# fdsn services
for s, v in node_par['services']['fdsn'].items():
_service = _lookup_service(_services, s, 'fdsn')
if v and _service:
logger.debug("Adding service '{}' to '{}'".format(_service, n))
n.services.append(_service)

# eida services
for s, v in node_par['services']['eida'].items():
_service = _lookup_service(_services, s, 'eida')
if v and _service:
logger.debug("Adding service '{}' to '{}'".format(_service, n))
n.services.append(_service)

except KeyError as err:
raise MissingNodeConfigParam(err)

_nodes.append(n)

# create endpoints and add service
try:
# fdsn services
for s, v in node_par['services']['fdsn'].items():
if v and _lookup_service(_services, s, 'fdsn'):
e = orm.Endpoint(
url='{}/fdsnws/{}/1/query'.format(
node_par['services']['fdsn']['server'], s),
service=_lookup_service(_services, s, 'fdsn'))

logger.debug('Created endpoint %r' % e)
_endpoints.append(e)

# eida services
for s, v in node_par['services']['eida'].items():
if (v['server'] and
_lookup_service(_services, s, 'eida')):
e = orm.Endpoint(
url='{}{}'.format(v['server'],
v['uri_path_query']),
service=_lookup_service(_services, s, 'eida'))

logger.debug('Created endpoint %r' % e)
_endpoints.append(e)

except KeyError as err:
raise InvalidEndpointConfig(err)

payload = _nodes
payload.extend(_endpoints)

with session_guard(session) as s:
s.add_all(payload)

# init ()

def clean(session, timestamp):
"""
Expand Down Expand Up @@ -261,9 +150,8 @@ def clean(session, timestamp):

vnets_active = set(
session.query(orm.StreamEpochGroup).\
filter(orm.StreamEpochGroup.oid.in_(vnets_active)).\
all())

filter(orm.StreamEpochGroup.oid.in_(vnets_active)).\
all())

vnets_all = set(session.query(orm.StreamEpochGroup).all())

Expand Down
78 changes: 24 additions & 54 deletions eidangservices/stationlite/engine/orm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import datetime

from sqlalchemy import (Column, Integer, Float, String, Unicode, DateTime,
ForeignKey, Table)
ForeignKey)
from sqlalchemy.ext.declarative import declared_attr, declarative_base
from sqlalchemy.orm import relationship

Expand All @@ -56,11 +56,12 @@ def __tablename__(cls):

# class Base


class EpochMixin(object):

@declared_attr
def starttime(cls):
return Column(DateTime, nullable=False, index=True)
return Column(DateTime, nullable=False, index=True)

@declared_attr
def endtime(cls):
Expand All @@ -82,48 +83,12 @@ def lastseen(cls):
# -----------------------------------------------------------------------------
ORMBase = declarative_base(cls=Base)

node_service_relation = Table(
'node_service_relation', ORMBase.metadata,
Column('node_ref', Integer, ForeignKey('service.oid')),
Column('service_ref', Integer, ForeignKey('node.oid')))


class Node(ORMBase):

oid = Column(Integer, primary_key=True)
name = Column(String(LENGTH_STD_CODE), nullable=False, unique=True)
description = Column(Unicode(LENGTH_DESCRIPTION))

networks = relationship('NodeNetworkInventory', back_populates='node')
services = relationship('Service',
secondary=node_service_relation,
back_populates='nodes')

def __repr__(self):
return '<Node(name=%s, description=%s)>' % (self.name,
self.description)

# class Node


class NodeNetworkInventory(LastSeenMixin, ORMBase):

# association object pattern
node_ref = Column(Integer, ForeignKey('node.oid'))
network_ref = Column(Integer, ForeignKey('network.oid'))

node = relationship('Node', back_populates='networks')
network = relationship('Network', back_populates='nodes')

# class NodeNetworkInventory


class Network(ORMBase):

name = Column(String(LENGTH_STD_CODE), nullable=False, index=True)

network_epochs = relationship('NetworkEpoch', back_populates='network')
nodes = relationship('NodeNetworkInventory', back_populates='network')
channel_epochs = relationship('ChannelEpoch',
back_populates='network')
stream_epochs = relationship('StreamEpoch', back_populates='network')
Expand All @@ -136,7 +101,8 @@ def __repr__(self):

class NetworkEpoch(EpochMixin, LastSeenMixin, ORMBase):

network_ref = Column(Integer, ForeignKey('network.oid'))
network_ref = Column(Integer, ForeignKey('network.oid'),
index=True)
description = Column(Unicode(LENGTH_DESCRIPTION))

network = relationship('Network', back_populates='network_epochs')
Expand All @@ -146,8 +112,10 @@ class NetworkEpoch(EpochMixin, LastSeenMixin, ORMBase):

class ChannelEpoch(EpochMixin, LastSeenMixin, ORMBase):

network_ref = Column(Integer, ForeignKey('network.oid'))
station_ref = Column(Integer, ForeignKey('station.oid'))
network_ref = Column(Integer, ForeignKey('network.oid'),
index=True)
station_ref = Column(Integer, ForeignKey('station.oid'),
index=True)
channel = Column(String(LENGTH_CHANNEL_CODE), nullable=False,
index=True)
locationcode = Column(String(LENGTH_LOCATION_CODE), nullable=False,
Expand Down Expand Up @@ -187,7 +155,8 @@ def __repr__(self):

class StationEpoch(EpochMixin, LastSeenMixin, ORMBase):

station_ref = Column(Integer, ForeignKey('station.oid'))
station_ref = Column(Integer, ForeignKey('station.oid'),
index=True)
description = Column(Unicode(LENGTH_DESCRIPTION))
longitude = Column(Float, nullable=False, index=True)
latitude = Column(Float, nullable=False, index=True)
Expand All @@ -199,8 +168,10 @@ class StationEpoch(EpochMixin, LastSeenMixin, ORMBase):

class Routing(EpochMixin, LastSeenMixin, ORMBase):

channel_epoch_ref = Column(Integer, ForeignKey('channelepoch.oid'))
endpoint_ref = Column(Integer, ForeignKey('endpoint.oid'))
channel_epoch_ref = Column(Integer, ForeignKey('channelepoch.oid'),
index=True)
endpoint_ref = Column(Integer, ForeignKey('endpoint.oid'),
index=True)

channel_epoch = relationship('ChannelEpoch', back_populates='endpoints')
endpoint = relationship('Endpoint', back_populates='channel_epochs')
Expand All @@ -214,7 +185,8 @@ def __repr__(self):

class Endpoint(ORMBase):

service_ref = Column(Integer, ForeignKey('service.oid'))
service_ref = Column(Integer, ForeignKey('service.oid'),
index=True)
url = Column(String(LENGTH_URL), nullable=False)

# many to many ChannelEpoch<->Endpoint
Expand All @@ -231,16 +203,11 @@ def __repr__(self):
class Service(ORMBase):

name = Column(String(LENGTH_STD_CODE), nullable=False, unique=True)
standard = Column(String(LENGTH_STD_CODE), nullable=False)

endpoints = relationship('Endpoint', back_populates='service')

nodes = relationship('Node',
secondary=node_service_relation,
back_populates='services')

def __repr__(self):
return '<Service(name=%s, standard=%s)>' % (self.name, self.standard)
return '<Service(name=%s)>' % self.name

# class Service

Expand All @@ -261,10 +228,13 @@ def __repr__(self):
# elegantly
class StreamEpoch(EpochMixin, LastSeenMixin, ORMBase):

network_ref = Column(Integer, ForeignKey('network.oid'))
station_ref = Column(Integer, ForeignKey('station.oid'))
network_ref = Column(Integer, ForeignKey('network.oid'),
index=True)
station_ref = Column(Integer, ForeignKey('station.oid'),
index=True)
stream_epoch_group_ref = Column(Integer,
ForeignKey('streamepochgroup.oid'))
ForeignKey('streamepochgroup.oid'),
index=True)
channel = Column(String(LENGTH_CHANNEL_CODE), nullable=False,
index=True)
location = Column(String(LENGTH_LOCATION_CODE), nullable=False,
Expand Down
Binary file renamed ...lite/example/db/stationlite_2018-03-21.db → ...lite/example/db/stationlite_2018-05-01.db
View file
Binary file not shown.
Loading

0 comments on commit c0fd56f

Please sign in to comment.