Skip to content

Commit

Permalink
Merge pull request #55 from scossu/development
Browse files Browse the repository at this point in the history
Alpha 13.
  • Loading branch information
scossu committed Apr 14, 2018
2 parents a1e534a + 84fa563 commit e71a132
Show file tree
Hide file tree
Showing 43 changed files with 504 additions and 242 deletions.
25 changes: 25 additions & 0 deletions .github/release_template.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# What Changed In This Release

[High-level, colloquial summary of changes—mandatory]

## New Features

-

## Enhancements

-

## Bug Fixes

-

## Other Changes

-

## Notes & Caveats

-

## Acknowledgments
3 changes: 1 addition & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -104,5 +104,4 @@ venv.bak/
.mypy_cache/

# Default LAKEsuperior data directories
data/ldpnr_store
data/ldprs_store
/data
1 change: 1 addition & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,6 @@ deploy:
on:
tags: true
branch: master
python: "3.5"
distributions: "bdist_wheel"

1 change: 1 addition & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
include README.rst
include LICENSE
graft lakesuperior/data/bootstrap
graft lakesuperior/endpoints/templates
graft lakesuperior/etc.defaults
1 change: 1 addition & 0 deletions VERSION
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1.0.0a13
42 changes: 27 additions & 15 deletions conftest.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,31 @@
import sys

import pytest

sys.path.append('.')
from lakesuperior.config_parser import test_config
from os import makedirs, path
from shutil import rmtree
from tempfile import gettempdir

from lakesuperior import env
from lakesuperior.config_parser import parse_config
from lakesuperior.globals import AppGlobals
from lakesuperior.env import env
from lakesuperior.util.generators import random_image

env.config = test_config
env.app_globals = AppGlobals(test_config)

# Override data directory locations.
config = parse_config()
data_dir = path.join(gettempdir(), 'lsup_test', 'data')
config['application']['data_dir'] = data_dir
config['application']['store']['ldp_nr']['location'] = (
path.join(data_dir, 'ldpnr_store'))
config['application']['store']['ldp_rs']['location'] = (
path.join(data_dir, 'ldprs_store'))

env.app_globals = AppGlobals(config)
from lakesuperior.app import create_app
from lakesuperior.util.generators import random_image

env.config = test_config

@pytest.fixture(scope='module')
def app():
app = create_app(env.config['application'])
app = create_app(env.app_globals.config['application'])

yield app

Expand All @@ -26,14 +35,17 @@ def db(app):
'''
Set up and tear down test triplestore.
'''
rdfly = env.app_globals.rdfly
rdfly.bootstrap()
makedirs(data_dir, exist_ok=True)
env.app_globals.rdfly.bootstrap()
env.app_globals.nonrdfly.bootstrap()
print('Initialized data store.')

yield rdfly
yield env.app_globals.rdfly

print('Tearing down fixture graph store.')
rdfly.store.destroy(rdfly.store.path)
# TODO improve this by using tempfile.TemporaryDirectory as a context
# manager.
print('Removing fixture data directory.')
rmtree(data_dir)


@pytest.fixture
Expand Down
Empty file removed data/log/.keep
Empty file.
Empty file removed data/run/.keep
Empty file.
2 changes: 1 addition & 1 deletion docs/apidoc/lakesuperior.rst
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ lakesuperior\.config\_parser module
lakesuperior\.env module
------------------------

.. automodule:: lakesuperior.env
.. automodule:: lakesuperior
:members:
:undoc-members:
:show-inheritance:
Expand Down
76 changes: 67 additions & 9 deletions docs/cli.rst
Original file line number Diff line number Diff line change
@@ -1,13 +1,39 @@
LAKEsuperior Command Line Reference
===================================
Command Line Reference
======================

The LAKEsuperior command line tool is used for maintenance and
administration purposes.
LAKEsuperior comes with some command-line tools aimed at several purposes.

The script should be in your executable path if you install LAKEsuperior with
``pip``. The tool is self-documented, so this is just a redundant overview::
If LAKEsuperior is installed via ``pip``, all tools can be invoked as normal
commands (i.e. they are in the virtualenv ``PATH``).

$ lsup_admin
The tools are currently not directly available on Docker instances (*TODO add
instructions and/or code changes to access them*).

``fcrepo``
----------

This is the main server command. It has no parameters. The command spawns
Gunicorn workers (as many as set up in the configuration) and can be sent in
the background, or started via init script.

The tool must be run in the same virtual environment LAKEsuperior
was installed in (if it was)—i.e.::

source <virtualenv root>/bin/activate

must be run before running the server.

In the case an init script is used, ``coilmq`` (belonging to a 3rd party
package) needs to be launched as well; unless a message broker is already set
up, or if messaging is disabled in the configuration.

``lsup-admin``
--------------

``lsup-admin`` is the principal repository management tool. It is
self-documented, so this is just a redundant overview::

$ lsup-admin
Usage: lsup-admin [OPTIONS] COMMAND [ARGS]...

Options:
Expand All @@ -21,8 +47,6 @@ The script should be in your executable path if you install LAKEsuperior with
migrate Migrate an LDP repository to LAKEsuperior.
stats Print repository statistics.

*TODO: Add instructions to access from Docker.*

All entries marked ``[STUB]`` are not yet implemented, however the
``lsup_admin <command> --help`` command will issue a description of what
the command is meant to do. Check the
Expand All @@ -31,3 +55,37 @@ the radar.

All of the above commands are also available via, and based upon, the
native Python API.

``lsup-benchmark``
------------------

``lsup-benchmark`` is used to run performance tests in a predictable way.

The command has no options but prompts the user for a few settings
interactively (N.B. this may change in favor of parameters).

The benchmark tool is able to create RDF sources, or non-RDF, or an equal mix
of them, via POST or PUT, in the currently running LAKEsuperior server. It
runs single-threaded.

The RDF sources are randomly generated graphs of consistent size and
complexity. They include a mix of in-repository references, literals, and
external URIs. Each graph has 200 triples.

The non-RDF sources are randomly generated 1024x1024 pixel PNG images.

You are warmly encouraged to run the script and share the performance results (
*TODO add template for posting results*).

``profiler``
------------

This command launches a single-threaded HTTP server (Flask) on port 5000 that
logs profiling information. This is useful for analyzing application
performance.

For more information, consult the `Python profilers guide
<https://docs.python.org/3/library/profile.html>`__.

Do not launch this while a WSGI server (``fcrepo``) is already running, because
that also launches a Flask server on port 5000.
13 changes: 6 additions & 7 deletions docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,11 @@
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys

from os import path
from unittest.mock import MagicMock

#sys.path.append(os.path.abspath('../'))

class MockModule(MagicMock):
@classmethod
def __getattr__(cls, name):
Expand All @@ -37,6 +35,8 @@ def __getattr__(cls, name):
import lakesuperior.env_setup


here = path.abspath(path.dirname(__file__))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
Expand Down Expand Up @@ -74,10 +74,9 @@ def __getattr__(cls, name):
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.0-alpha'
# The full version, including alpha/beta/rc tags.
release = '1.0.0-alpha.8'
# Version and release are the same.
with open(path.realpath(path.join(here, '..', 'VERSION'))) as fh:
version = release = fh.readlines()[0]

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
Expand Down
20 changes: 14 additions & 6 deletions docs/setup.rst
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,14 @@ Configuration

The app should run for testing and evaluation purposes without any
further configuration. All the application data are stored by default in
the ``data`` directory.
the ``data`` directory of the Python package.

To change the default configuration you should:
This setup is not recommended for anything more than a quick look at the
application. If more complex interaction is needed, or upgrades to the package
are foreseen, it is strongly advised to set up proper locations for
configuration and data.

To change the default configuration you need to:

#. Copy the ``etc.default`` folder to a separate location
#. Set the configuration folder location in the environment:
Expand All @@ -94,10 +99,13 @@ To change the default configuration you should:

The configuration options are documented in the files.

**Note:** ``test.yml`` must specify a different location for the graph
and for the binary stores than the default one, otherwise running a test
suite will destroy your main data store. The application will issue an
error message and refuse to start if these locations overlap.
One thing worth noting is that some locations can be specified as relative
paths. These paths will be relative to the ``data_dir`` location specified in
the ``application.yml`` file.

If ``data_dir`` is empty, as it is in the default configuration, it defaults
to the ``data`` directory inside the Python package. This is the option that
one may want to change before anything else.

Production deployment
---------------------
Expand Down
5 changes: 3 additions & 2 deletions docs/usage.rst
Original file line number Diff line number Diff line change
Expand Up @@ -123,11 +123,12 @@ Before using the API, either do::

Or, to specify an alternative configuration::

>>> from lakesuperior import env
>>> from lakesuperior.config_parser import parse_config
>>> from lakesuperior.globals import AppGlobals
>>> env.config, test_config = parse_config('/my/custom/config_dir')
>>> config = parse_config('/my/custom/config_dir')
Reading configuration at /my/custom/config_dir
>>> env.app_globals = AppGlobals(env.config)
>>> env.app_globals = AppGlobals(config)

Create and replace resources
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Expand Down
49 changes: 49 additions & 0 deletions lakesuperior/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import threading

from os import path

basedir = path.dirname(path.realpath(__file__))
"""
Base directory for the module.
This can be used by modules looking for configuration and data files to be
referenced or copied with a known path relative to the package root.
:rtype: str
"""

class Env:
pass

env = Env()
"""
A pox on "globals are evil".
All-purpose bucket for storing global variables. Different environments
(e.g. webapp, test suite) put the appropriate value in it.
The most important values to be stored are app_conf (either from
lakesuperior.config_parser.config or lakesuperior.config_parser.test_config)
and app_globals (obtained by an instance of lakesuperior.globals.AppGlobals).
e.g.::
>>> from lakesuperior.config_parser import config
>>> from lakesuperior.globals import AppGlobals
>>> from lakesuperior import env
>>> env.app_globals = AppGlobals(config)
This is automated in non-test environments by importing
`lakesuperior.env_setup`.
:rtype: Object
"""

thread_env = threading.local()
"""
Thread-local environment.
This is used to store thread-specific variables such as start/end request
timestamps.
:rtype: threading.local
"""
12 changes: 3 additions & 9 deletions lakesuperior/api/admin.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import logging

from lakesuperior import env
from lakesuperior.config_parser import parse_config
from lakesuperior.env import env
from lakesuperior.globals import AppGlobals
from lakesuperior.migrator import Migrator
from lakesuperior.store.ldp_nr.default_layout import DefaultLayout as FileLayout
from lakesuperior.store.ldp_rs.lmdb_store import TxnManager
Expand Down Expand Up @@ -47,17 +46,12 @@ def migrate(src, dest, start_pts=None, list_file=None, **kwargs):
return Migrator(src, dest, **kwargs).migrate(start_pts, list_file)


def integrity_check(config_dir=None):
def integrity_check():
"""
Check integrity of the data set.
At the moment this is limited to referential integrity. Other checks can
be added and triggered by different argument flags.
"""
if config_dir:
env.config = parse_config(config_dir)[0]
env.app_globals = AppGlobals(env.config)
else:
import lakesuperior.env_setup
with TxnManager(env.app_globals.rdfly.store):
return { t for t in env.app_globals.rdfly.find_refint_violations()}
return set(env.app_globals.rdfly.find_refint_violations())
2 changes: 1 addition & 1 deletion lakesuperior/api/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@

from io import BytesIO

from lakesuperior import env
from lakesuperior.dictionaries.namespaces import ns_collection as nsc
from lakesuperior.dictionaries.namespaces import ns_mgr as nsm
from lakesuperior.env import env
from lakesuperior.store.ldp_rs.lmdb_store import LmdbStore, TxnManager


Expand Down

0 comments on commit e71a132

Please sign in to comment.