Packaging (#39)

Restructure many files for distro inclusion.

* Add assets and HTML templates
* Move utils under lakesuperior package

MANIFEST.in

@@ -0,0 +1,3 @@
+include README.rst
+include LICENSE
+graft lakesuperior/endpoints/templates

conftest.py

@@ -6,9 +6,11 @@
 from lakesuperior.config_parser import test_config
 from lakesuperior.globals import AppGlobals
 from lakesuperior.env import env
+
+env.config = test_config
 env.app_globals = AppGlobals(test_config)
 from lakesuperior.app import create_app
-from util.generators import random_image
+from lakesuperior.util.generators import random_image
 
 env.config = test_config
 

docs/setup.rst

@@ -46,17 +46,29 @@ Dependencies
 Installation steps
 ~~~~~~~~~~~~~~~~~~
 
-#. Create a virtualenv in a project folder:
-   ``python3 -m venv <virtualenv folder>``
-#. Activate the virtualenv: ``source <path_to_virtualenv>/bin/activate``
-#. Install dependencies: ``pip install -r requirements.txt``
-#. Start your STOMP broker, e.g.: ``coilmq &``. If you have another
-   queue manager listening to port 61613 you can either configure a
-   different port on the application configuration, or use the existing
-   message queue.
-#. Run ``lsup-admin bootstrap`` to initialize the binary and graph
-   stores.
-#. Run ``fcrepo``.
+Start in an empty project folder. If you are feeling lazy you can copy
+and paste the lines below in your console.
+
+::
+
+    mkdir lsup_env # Or whatever you want to call it
+    cd lsup_env
+    python3 -m venv .
+    source bin/activate
+    pip install lakesuperior
+    # Start the message broker. If you have another
+    # queue manager listening to port 61613 you can either configure a
+    # different port on the application configuration, or use the existing
+    # message queue.
+    coilmq&
+    # Bootstrap the repo
+    echo yes | lsup-admin bootstrap
+    # Run the thing
+    fcrepo
+
+Test if it works::
+
+    curl http://localhost:8000/ldp/
 
 Configuration
 -------------

lakesuperior/endpoints/ldp.py

@@ -37,7 +37,7 @@
 
 ldp = Blueprint(
         'ldp', __name__, template_folder='templates',
-        static_url_path='/static', static_folder='../../static')
+        static_url_path='/static', static_folder='templates/static')
 
 accept_patch = (
     'application/sparql-update',

lakesuperior/endpoints/main.py

@@ -7,7 +7,7 @@
 # Blueprint for main pages. Not much here.
 
 main = Blueprint('main', __name__, template_folder='templates',
-        static_folder='../../static')
+        static_folder='templates/static')
 
 ## GENERIC ROUTES ##
 

lakesuperior/util/benchmark.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+import sys
+sys.path.append('.')
+
+from uuid import uuid4
+
+import arrow
+import requests
+
+from lakesuperior.util.generators import (
+        random_image, random_graph, random_utf8_string)
+
+__doc__ = '''
+Benchmark script to measure write performance.
+'''
+
+default_n = 10000
+webroot = 'http://localhost:8000/ldp'
+container_uri = webroot + '/pomegranate'
+
+def run():
+    sys.stdout.write('How many children? [{}] >'.format(default_n))
+    choice = input().lower()
+    n = int(choice) if choice else default_n
+
+    sys.stdout.write('Delete container? [n] >')
+    choice = input().lower()
+    del_cont = choice or 'n'
+
+    sys.stdout.write('POST or PUT? [PUT] >')
+    choice = input().lower()
+    if choice and choice.lower() not in ('post', 'put'):
+        raise ValueError('Not a valid verb.')
+    method = choice.lower() or 'put'
+
+    sys.stdout.write('RDF Sources (r), Non-RDF (n), or Both 50/50 (b)? [b] >')
+    choice = input().lower()
+    res_type = choice or 'b'
+
+    if del_cont  == 'y':
+        requests.delete(container_uri, headers={'prefer': 'no-tombstone'})
+    requests.put(container_uri)
+
+    start = arrow.utcnow()
+    ckpt = start
+
+    print('Inserting {} children.'.format(n))
+
+    # URI used to establish an in-repo relationship.
+    ref = container_uri
+    size = 200 # Size of graph.
+
+    try:
+        for i in range(1, n + 1):
+            url = '{}/{}'.format(container_uri, uuid4()) if method == 'put' \
+                    else container_uri
+
+            if res_type == 'r' or (res_type == 'b' and i % 2 == 0):
+                data = random_graph(size, ref).serialize(format='ttl')
+                headers = {'content-type': 'text/turtle'}
+            else:
+                img = random_image(name=uuid4(), ts=16, ims=512)
+                data = img['content']
+                data.seek(0)
+                headers = {
+                        'content-type': 'image/png',
+                        'content-disposition': 'attachment; filename="{}"'
+                            .format(uuid4())}
+
+            #import pdb; pdb.set_trace()
+            rsp = requests.request(method, url, data=data, headers=headers)
+            rsp.raise_for_status()
+            ref = rsp.headers['location']
+            if i % 10 == 0:
+                now = arrow.utcnow()
+                tdelta = now - ckpt
+                ckpt = now
+                print('Record: {}\tTime elapsed: {}'.format(i, tdelta))
+    except KeyboardInterrupt:
+        print('Interrupted after {} iterations.'.format(i))
+
+    tdelta = arrow.utcnow() - start
+    print('Total elapsed time: {}'.format(tdelta))
+    print('Average time per resource: {}'.format(tdelta.total_seconds()/i))
+
+if __name__ == '__main__':
+    run()

wsgi.py → lakesuperior/wsgi.py

@@ -1,11 +1,11 @@
 import multiprocessing
 import yaml
 
-from os import environ, path
+from os import environ, makedirs, path
 
 import gunicorn.app.base
 
-from server import fcrepo
+from lakesuperior.server import fcrepo
 
 
 default_config_dir = '{}/etc.defaults'.format(
@@ -20,7 +20,12 @@
 listen_port = config.get('listen_port', 8000)
 preload_app = config.get('preload_app', True)
 app_mode = config.get('app_mode', 'prod')
+
 data_dir = path.realpath(config.get('data_dir'))
+run_dir = '{}/run'.format(data_dir)
+log_dir = '{}/log'.format(data_dir)
+makedirs(log_dir, exist_ok=True)
+makedirs(run_dir, exist_ok=True)
 
 def default_workers():
     return (multiprocessing.cpu_count() * 2) + 1
@@ -40,12 +45,11 @@ def default_workers():
     'daemon': app_mode=='prod',
     'reload': app_mode=='dev' and not preload_app,
 
-    'pidfile': '{}/run/fcrepo.pid'.format(data_dir),
-    'accesslog': '{}/log/gunicorn-access.log'.format(data_dir),
-    'errorlog': '{}/log/gunicorn-error.log'.format(data_dir),
+    'pidfile': '{}/fcrepo.pid'.format(run_dir),
+    'accesslog': '{}/gunicorn-access.log'.format(log_dir),
+    'errorlog': '{}/gunicorn-error.log'.format(log_dir),
 }
 
-
 class WsgiApp(gunicorn.app.base.BaseApplication):
 
     def __init__(self, app, options={}):

setup.py

@@ -10,6 +10,7 @@
 from setuptools import setup, find_packages
 # To use a consistent encoding
 from codecs import open
+from glob import glob
 from os import path
 
 here = path.abspath(path.dirname(__file__))
@@ -26,7 +27,7 @@
 
 setup(
     name='lakesuperior',
-    version='1.0.0a91',
+    version='1.0.0a98',
 
     description='A Linked Data Platform repository sever.',
     long_description=long_description,
@@ -100,15 +101,24 @@
         'pytest-flask',
     ],
 
+    include_package_data=True,
     #extras_require={},
-    #package_data={},
-    #data_files=[],
+    #package_data={
+    #    'endpoints': ['templates/*'],
+    #},
+    data_files=[
+        ('etc', glob('etc.defaults/*.yml')),
+        ('data/bootstrap', glob('data/bootstrap/*')),
+        ('data/ldpnr_store', ['data/ldpnr_store/.keep']),
+        ('data/ldprs_store', ['data/ldprs_store/.keep']),
+    ],
 
     entry_points={
         'console_scripts': [
-            'lsup-admin=lsup_admin:admin',
-            'profiler=profiler:run',
-            'fcrepo=wsgi:run',
+            'fcrepo=lakesuperior.wsgi:run',
+            'lsup-admin=lakesuperior.lsup_admin:admin',
+            'lsup-benchmark=lakesuperior.util.benchmark:run',
+            'profiler=lakesuperior.profiler:run',
         ],
     },