Skip to content
Browse files

merge commit

  • Loading branch information...
2 parents d7cfb50 + 0da138d commit 6c68b5c7d15fcff5cd9c33169965f6edd34e4356 @matley committed May 18, 2012
View
6 .gitignore
@@ -3,9 +3,11 @@
nosetest*
.coverage
*~
-#*#
+*#*#
.#*
*.db-journal
.ropeproject
*_flymake.py
-tests/data/actual*
+tests/data/actual*
+docs/build
+
View
216 docs/source/conf.py
@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+#
+# oq-eqcatalogue-tool documentation build configuration file, created by
+# sphinx-quickstart on Thu May 17 14:35:15 2012.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.pngmath']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'oq-eqcatalogue-tool'
+copyright = u'2012, GEM'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '0.1'
+# The full version, including alpha/beta/rc tags.
+release = version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+html_theme = 'sphinxdoc'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'oq-eqcatalogue-tooldoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+# The paper size ('letter' or 'a4').
+#latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+#latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+ ('index', 'oq-eqcatalogue-tool.tex', u'oq-eqcatalogue-tool Documentation',
+ u'GEM', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Additional stuff for the LaTeX preamble.
+#latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ ('index', 'oq-eqcatalogue-tool', u'oq-eqcatalogue-tool Documentation',
+ [u'GEM'], 1)
+]
View
40 docs/source/dev/requirements.rst
@@ -0,0 +1,40 @@
+.. _requirements:
+
+Software dependencies
+==============================================================================
+
+This part of documentation covers the installation of ECT.
+ECT `Python 2.7.x`_ version of the interpreter and `Spatialite 3.0.x`_.
+The first step is getting external dependencies properly installed.
+ECT requires these libraries in order to be used:
+
+* GeoAlchemy_ >= 0.7.x
+* SqlAlchemy_ >= 0.7.x
+* Scipy_ >= 0.10.x
+* Numpy_ >= 1.6.x
+* Matplotlib_ >= 1.1.x
+
+Instructions for setting up GeoAlchemy with Spatialite are provided here_.
+
+Get the code
+=============================================================================
+
+ECT is actively developed on GitHub, where the code is
+`available <https://github.com/gem/oq-eqcatalogue-tool>`_.
+You can clone the repository doing::
+
+ git clone git@github.com:gem/oq-eqcatalogue-tool.git
+
+Or download the
+`zip <https://github.com/gem/oq-eqcatalogue-tool/zipball/master>`_.
+
+
+.. Links
+.. _Python 2.7.x: http://www.python.org/getit/releases/2.7/
+.. _Spatialite 3.0.x: http://www.gaia-gis.it/gaia-sins/
+.. _GeoAlchemy: http://www.geoalchemy.org
+.. _SqlAlchemy: http://www.sqlalchemy.org/
+.. _Scipy: http://www.scipy.org/
+.. _Numpy: http://numpy.org/
+.. _Matplotlib: http://matplotlib.sourceforge.net/
+.. _here: http://www.geoalchemy.org/usagenotes.html#notes-for-spatialite
View
34 docs/source/index.rst
@@ -0,0 +1,34 @@
+.. oq-eqcatalogue-tool documentation master file, created by
+ sphinx-quickstart on Thu May 17 14:35:15 2012.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+Welcome to oq-eqcatalogue-tool's documentation!
+===============================================
+
+Release v\ |version|.
+
+EqCatalogue Tool (ECT) is a `Python`_ library which allows the user
+to homogenise an earthquake catalogue, useful in the context of the
+Openquake_ platform framework. ECT is written by GEM Foundation and
+licensed under `GNU Affero General Public License`_.
+
+Requirements
+===============================================
+.. toctree::
+ :maxdepth: 2
+
+ dev/requirements.rst
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
+.. Links
+.. _Python: http://www.python.org
+.. _OpenQuake: http://www.openquake.org/
+.. _GNU Affero General Public License:
+ http://www.gnu.org/licenses/agpl-3.0.html
View
15 eqcatalogue/events.py
@@ -34,8 +34,7 @@ def all(self):
all events inside the earthquake catalogue.
"""
- return self._session.query(db.Event).join(db.MagnitudeMeasure).join(
- db.Origin)
+ return self._session.query(db.Event)
def before(self, time):
"""
@@ -117,3 +116,15 @@ def within_distance_from_point(self, point, distance):
db.Origin).filter(
"PtDistWithin(catalogue_origin.position, GeomFromText('%s', "
"4326), %s)" % (point, distance))
+
+ @classmethod
+ def group_measures(cls, query_obj):
+ """
+ Group all measures by event
+ :param query_obj: sqlalchemy query object.
+ """
+
+ groups = []
+ for ev in query_obj.all():
+ groups.append(dict(event=ev, measures=ev.measures))
+ return groups
View
63 eqcatalogue/importers/isf_bulletin.py
@@ -16,11 +16,11 @@
import re
import urllib
import datetime
-import logging
from eqcatalogue import models as catalogue
CATALOG_URL = 'http://www.isc.ac.uk/cgi-bin/web-db-v4'
+
ANALYSIS_TYPES = {'a': 'automatic',
'm': 'manual',
'g': 'guess'}
@@ -31,6 +31,7 @@
'g': 'ground truth',
'o': 'other'
}
+
EVENT_TYPES = {
'uk': 'unknown',
'de': 'damaging earthquake ( Not standard IMS )',
@@ -54,6 +55,9 @@
class UnexpectedLine(BaseException):
+ """
+ Exception raised when an unexpected line input is found
+ """
def __init__(self, state, line_type):
super(UnexpectedLine, self).__init__()
self.state = state
@@ -66,7 +70,16 @@ def __str__(self):
self.state, self.line_type, self.line, len(self.line))
+# Imp. Notes. Parsing is done by using a FSM. Each line has a
+# line_type which acts as an "event" and it is an instance of a
+# particular State
+
class BaseState(object):
+ """
+ The base state object. A state stores the catalogue db instance
+ and calculate the next state based on the current event
+ """
+
def __init__(self):
self._catalogue = None
@@ -76,18 +89,27 @@ def setup(self, cat):
def is_start(self):
return False
- def transitionRule(self, line_type):
+ def transition_rule(self, line_type):
next_state = self._get_next_state(line_type)
if not next_state:
raise UnexpectedLine(self, line_type)
else:
return next_state
def process_line(self, _):
+ """
+ When a state is initialized, this function is called. It
+ actually parses the line content and eventually creates the
+ proper model object. It returns a dictionary with the partial
+ summary of this import phase
+ """
return {}
class StartState(BaseState):
+ """
+ Start State. The FSM is initialized with this state
+ """
def __init__(self):
super(StartState, self).__init__()
self.eventsource = None
@@ -111,6 +133,10 @@ def _save_eventsource(self, name):
class EventState(BaseState):
+ """
+ When data about a seismic event arrives, the fsm jumps to an Event
+ State
+ """
def __init__(self, eventsource):
super(EventState, self).__init__()
self._eventsource = eventsource
@@ -122,6 +148,8 @@ def _get_next_state(self, line_type):
@classmethod
def match(cls, line):
+ """Return True if line match a proper regexp, that triggers an
+ event that makes the fsm jump to an EventState"""
event_regexp = re.compile(
'^Event (?P<source_event_id>\w{0,9}) (?P<name>.{0,65})$')
return event_regexp.match(line)
@@ -305,6 +333,9 @@ def _save_origin(self, source_key, **kwargs):
class MeasureBlockState(BaseState):
+ """
+ When a Measure Block is found the fsm jumps to this state
+ """
def __init__(self, event, metadata):
super(MeasureBlockState, self).__init__()
self.event = event
@@ -364,9 +395,15 @@ def _save_metadata(self, stations):
class MeasureUKScaleBlockState(MeasureBlockState):
+ """
+ When a Measure Block with an unknown scale is found the fsm jumps
+ to this state
+ """
+
@classmethod
def match(cls, line):
- pat = '^(?P<val>-*[0-9]+\.[0-9]+)\s+(?P<error>[0-9]+\.[0-9]+)*\s+(?P<stations>[0-9]+)*\s+(?P<agency>\w+)\s+(?P<origin>\w+)$'
+ pat = ('^(?P<val>-*[0-9]+\.[0-9]+)\s+(?P<error>[0-9]+\.[0-9]+)*\s+'
+ '(?P<stations>[0-9]+)*\s+(?P<agency>\w+)\s+(?P<origin>\w+)$')
return re.compile(pat).match(line)
def process_line(self, line):
@@ -383,16 +420,32 @@ def process_line(self, line):
class V1(object):
- log = logging.getLogger(__name__)
+ """
+ The main class implementing the FSM.
+ """
def __init__(self, stream, cat):
+ """
+ Initialize the FSM.
+
+ :py:param:: stream
+ A stream object storing the seismic event data
+
+ :py:param:: cat
+ The catalogue database used to import the data
+ """
self._stream = stream
self._catalogue = cat
self._summary = {}
self._state = None
self._transition(StartState())
def load(self, allow_junk=True):
+ """
+ Read and parse from the input stream the data and insert them
+ into the catalogue db. If allow_junk is True, it allows
+ unexpected line inputs at the beginning of the file
+ """
for line in self._stream:
line = line.strip()
@@ -407,7 +460,7 @@ def load(self, allow_junk=True):
break
try:
- next_state = self._state.transitionRule(line_type)
+ next_state = self._state.transition_rule(line_type)
self._transition(next_state)
state_output = next_state.process_line(line)
self._update_summary(state_output)
View
4 eqcatalogue/models.py
@@ -231,7 +231,9 @@ class Origin(object):
together with `source_key`
"""
def __repr__(self):
- return "%s@%s at %s" % (self.position, self.time, self.depth)
+ return "id:%s eventsource:%s sourcekey:%s" % (self.id,
+ self.eventsource,
+ self.source_key)
def __init__(self, position, time, eventsource, source_key,
**kwargs):
View
15 tests/test_filtering.py
@@ -76,7 +76,7 @@ def load_fixtures(session):
session.add(measure_meta)
-class AnEqCatalogueShould(unittest.TestCase):
+class AnEventManagerShould(unittest.TestCase):
def setUp(self):
self.cat_db = models.CatalogueDatabase(memory=False, drop=True)
@@ -85,7 +85,7 @@ def setUp(self):
load_fixtures(self.session)
def test_allows_selection_of_all_events(self):
- self.assertEqual(30, self.event.all().count())
+ self.assertEqual(5, len(self.event.all().all()))
def test_allows_selection_events_on_time_criteria(self):
time = datetime.now()
@@ -144,5 +144,16 @@ def test_allows_selection_of_events_given_distance_from_point(self):
self.assertEqual(5, len(self.event.within_distance_from_point(point,
distance).all()))
+ def test_allows_grouping_of_measures(self):
+ all_events = self.event.all()
+ groups = EventManager.group_measures(all_events)
+
+ self.assertEqual(5, len(groups))
+ self.assertEqual(6, len(groups[0]['measures']))
+ self.assertEqual(13, len(groups[1]['measures']))
+ self.assertEqual(1, len(groups[2]['measures']))
+ self.assertEqual(6, len(groups[3]['measures']))
+ self.assertEqual(4, len(groups[4]['measures']))
+
def tearDown(self):
self.session.commit()
View
40 tests/test_importers.py
@@ -13,16 +13,11 @@
# You should have received a copy of the GNU Affero General Public License
# along with eqcatalogueTool. If not, see <http://www.gnu.org/licenses/>.
-import os
import unittest
from eqcatalogue.importers import isf_bulletin as isf
from eqcatalogue import models as catalogue
+from tests.test_utils import in_data_dir
-DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
-
-
-def in_data_dir(filename):
- return os.path.join(DATA_DIR, filename)
# the following data has been downloaded by issuing the
# following command
@@ -34,26 +29,27 @@ def in_data_dir(filename):
class ShouldImportFromISFBulletinV1(unittest.TestCase):
+ def setUp(self):
+ self.f = file(DATAFILE)
+ self.cat = catalogue.CatalogueDatabase(memory=True, drop=True)
+
+ def tearDown(self):
+ self.f.close()
+
def test_detect_junk_lines(self):
- # Assess
- f = file(DATAFILE)
- cat = catalogue.CatalogueDatabase(memory=True, drop=True)
+ # Common Assess part in setUp method
# Act
- importer = isf.V1(f, cat)
+ importer = isf.V1(self.f, self.cat)
# Assert
self.assertRaises(isf.UnexpectedLine, importer.load, (False))
- f.close()
-
def test_parse_html_file(self):
- # Assess
- f = file(DATAFILE)
- cat = catalogue.CatalogueDatabase(memory=True, drop=True)
+ # Common Assess part in setUp method
# Act
- summary = isf.V1.import_events(f, cat)
+ summary = isf.V1.import_events(self.f, self.cat)
# Assert
self.assertEqual(summary, {
@@ -64,16 +60,14 @@ def test_parse_html_file(self):
'measure_created': 5091,
})
- sources = cat.session.query(catalogue.EventSource)
- agencies = cat.session.query(catalogue.Agency)
- events = cat.session.query(catalogue.Event)
- origins = cat.session.query(catalogue.Origin)
- measures = cat.session.query(catalogue.MagnitudeMeasure)
+ sources = self.cat.session.query(catalogue.EventSource)
+ agencies = self.cat.session.query(catalogue.Agency)
+ events = self.cat.session.query(catalogue.Event)
+ origins = self.cat.session.query(catalogue.Origin)
+ measures = self.cat.session.query(catalogue.MagnitudeMeasure)
self.assertEqual(sources.count(), 1)
self.assertEqual(agencies.count(), 75)
self.assertEqual(events.count(), 1254)
self.assertEqual(origins.count(), 2770)
self.assertEqual(measures.count(), 5091)
-
- f.close()
View
4 tests/test_utils.py
@@ -24,3 +24,7 @@ def get_data_path(data_dir, filename):
"""Return the data path of the file used in test."""
return os.path.join(data_dir, filename)
+
+
+def in_data_dir(filename):
+ return get_data_path(DATA_DIR, filename)

0 comments on commit 6c68b5c

Please sign in to comment.
Something went wrong with that request. Please try again.