Permalink
Browse files

Updated data services prototype

  • Loading branch information...
1 parent d0fa988 commit 8808861caf1d2f4b3ca130d80d255aeda2d695e6 @wegrata wegrata committed Mar 8, 2012
View
44 LR/LR.egg-info/SOURCES.txt
@@ -1,5 +1,4 @@
MANIFEST.in
-README.txt
setup.cfg
setup.py
LR.egg-info/._PKG-INFO
@@ -35,7 +34,7 @@ lr/controllers/publish.py
lr/controllers/services.py
lr/controllers/slice.py
lr/controllers/status.py
-lr/controllers/sword.py
+lr/controllers/swordservice.py
lr/controllers/contrib/__init__.py
lr/controllers/contrib/filters.py
lr/lib/__init__.py
@@ -47,6 +46,14 @@ lr/lib/model_parser.py
lr/lib/model_parser1.py
lr/lib/oaipmh.py
lr/lib/oaipmherrors.py
+lr/lib/resumption_token.py
+lr/lib/stream.py
+lr/lib/couch_change_monitor/__init__.py
+lr/lib/couch_change_monitor/base_change_handler.py
+lr/lib/couch_change_monitor/base_change_monitor.py
+lr/lib/couch_change_monitor/base_change_threshold_handler.py
+lr/lib/couch_change_monitor/base_views_update_handler.py
+lr/lib/couch_change_monitor/change_monitor.py
lr/model/__init__.py
lr/model/base_model.py
lr/model/community.py
@@ -59,11 +66,33 @@ lr/model/node_filter.py
lr/model/node_service.py
lr/model/node_status.py
lr/model/resource_data.py
+lr/model/resource_data_monitor/__init__.py
+lr/model/resource_data_monitor/distributable_handler.py
+lr/model/resource_data_monitor/distribute_threshold_handler.py
+lr/model/resource_data_monitor/resource_data_handler.py
+lr/model/resource_data_monitor/track_last_sequence.py
+lr/model/resource_data_monitor/update_views_handler.py
+lr/mustache/__init__.py
+lr/mustache/oaipmh.py
lr/public/LRlogo.png
lr/public/bg.png
lr/public/favicon.ico
lr/public/index.html
lr/public/pylons-logo.gif
+lr/public/repofringe/RepoFringe 2011 Demo Proposal.pdf
+lr/public/repofringe/RepoFringe HackDay Jim Klo.pptx
+lr/public/repofringe/app.css
+lr/public/repofringe/app.js
+lr/public/repofringe/app.less
+lr/public/repofringe/excanvas.js
+lr/public/repofringe/id.js
+lr/public/repofringe/index.html
+lr/public/repofringe/jit-yc.js
+lr/public/repofringe/jit.js
+lr/public/repofringe/jquery-ui.css
+lr/public/repofringe/jquery-ui.min.js
+lr/public/repofringe/jquery.mustache.js
+lr/public/repofringe/less-1.1.3.min.js
lr/public/schemas/OAI/2.0/OAI-PMH-LR.xsd
lr/public/schemas/OAI/2.0/OAI-PMH.xsd
lr/public/schemas/OAI/2.0/sample-GetRecord-list.xml
@@ -83,18 +112,15 @@ lr/tests/functional/__init__.py
lr/tests/functional/test_OAI-PMH.py
lr/tests/functional/test_contrib_filters.py
lr/tests/functional/test_description.py
-lr/tests/functional/test_distribute.py
lr/tests/functional/test_harvest.py
lr/tests/functional/test_obtain.py
lr/tests/functional/test_policy.py
lr/tests/functional/test_publish.py
lr/tests/functional/test_services.py
-lr/tests/functional/test_setup.py
lr/tests/functional/test_slices.py
lr/tests/functional/test_status.py
lr/tests/functional/test_sword.py
-tests/__init__.py
-tests/headers.py
-tests/test_external.py
-tests/test_oaipmh.py
-tests/test_sword.py
+lr/util/__init__.py
+lr/util/decorators.py
+lr/util/testdata.py
+lr/util/validator.py
View
17 LR/LR.egg-info/requires.txt
@@ -1,6 +1,11 @@
-Pylons>=0.10
-pyparsing>=1.5.5
-restkit>=3.2.3
-couchdb>=0.8
-lxml>=2.3
-iso8601plus
+WebOb==1.1.1
+Pylons==1.0
+pyparsing==1.5.5
+restkit==3.2.3
+couchdb==0.8
+lxml==2.3
+iso8601plus==0.1.5
+ijson==0.8.0
+pystache==0.3.1
+PyJWT==0.1.4
+Couchapp==0.8.1
View
1 LR/LR.egg-info/top_level.txt
@@ -1,2 +1 @@
-tests
lr
View
3 LR/lr/config/routing.py
@@ -55,7 +55,8 @@ def mapResource(config_key, member_name, collection_name):
mapResource('lr.oaipmh.docid', 'OAI-PMH', 'OAI-PMH')
mapResource('lr.slice.docid', 'slice', 'slice')
mapResource('lr.sword.docid', 'swordservice','swordservice')
- map.connect("/extract/{view}/{param}",controller='extract', action='get')
+ map.connect("/extract/{dataservice}/{view}",controller='extract', action='get')
+ map.connect("/extract//{dataservice}/{view}/format/{show}",controller='extract', action='get')
map.minimization = False
map.explicit = False
View
57 LR/lr/controllers/extract.py
@@ -1,5 +1,6 @@
import logging
-
+from iso8601 import parse_date
+from datetime import datetime
from pylons import request, response, session, tmpl_context as c, url
from pylons.controllers.util import abort, redirect
from lr.model.base_model import appConfig
@@ -15,17 +16,17 @@ class ExtractController(BaseController):
# To properly map this controller, ensure your config/routing.py
# file has a resource setup:
# map.resource('extract', 'extract')
- def _getView(self,view='_all_docs',keys=[], includeDocs=True):
+ def _getView(self,view='_all_docs',keys=[], includeDocs=True,startKey=None,endKey=None):
args = {'include_docs':includeDocs}
if len(keys) > 0:
args['keys'] = keys
- args['limit'] = 10
+ args['reduce']=False
+ if startKey is not None:
+ args['startkey'] = startKey
+ if endKey is not None:
+ args['endkey'] = endKey
db_url = '/'.join([appConfig['couchdb.url'],appConfig['couchdb.db.resourcedata']])
- if includeDocs:
- documentHandler = lambda d: d['doc']
- else:
- documentHandler = lambda d: d['id']
- view = h.getView(database_url=db_url,view_name=view,documentHandler=documentHandler,**args)
+ view = h.getView(database_url=db_url,view_name=view,documentHandler=h.document,**args)
return view
def _streamList(self,data,dataHandler):
first = True
@@ -44,16 +45,40 @@ def _getBaseResult(self,data={},ids=[], includeDocs=True):
})
index = base.rfind('[') + 1
yield base[:index]
- for d in self._streamList(self._getView(includeDocs=False),json.dumps):
+ for d in self._streamList(self._getView(keys=ids,includeDocs=includeDocs),lambda d: json.dumps(d)):
yield d
yield base[index:]
- def get(self, view='',param=''):
- """GET /extract/id: Show a specific item"""
- yield '{"documents":['
- first = True
+ def _convertDateTime(self,dt):
+ if isinstance(dt,str):
+ dt = parse_date(dt)
+ dt = dt - datetime(1970,1,1)
+ return dt.total_seconds()
+ def _processRequest(self,startKey, endKey,urlBase):
+ yield '{"documents":['
def processFeed(item):
- for piece in self._getBaseResult():
+ for piece in self._getBaseResult(item,ids=[item['id']]):
yield piece
- for d in self._streamList(xrange(10),processFeed):
+ for d in self._streamList(self._getView(urlBase,startKey=startKey,endKey=endKey,includeDocs=False),processFeed):
yield d
- yield ']}'
+ yield ']}'
+ def get(self, dataservice="",view=''):
+ """GET /extract/id: Show a specific item"""
+
+ startKey=[]
+ endKey=[]
+ urlBase = "_design/{0}/_view/{1}".format(dataservice,view)
+ if 'from' in request.params:
+ startKey.append(self._convertDateTime(request.params['from']))
+ else:
+ startKey.append(self._convertDateTime(datetime.min))
+ if 'until' in request.params:
+ endKey.append(self._convertDateTime(request.params['until']))
+ else:
+ endKey.append(self._convertDateTime(datetime.max))
+ if 'disciminator' in request.params:
+ startKey.append(request.params['disciminator'])
+ endKey.append(response.params['disciminator']+'\ud7af')
+ else:
+ startKey.append('')
+ endKey.append('\ud7af')
+ return self._processRequest(startKey,endKey,urlBase)
View
2 data-pumps/adl/sync_dc.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-import couchdb,
+import couchdb
from oaipmh.client import Client
from oaipmh.common import Identify, Metadata, Header
from oaipmh.metadata import MetadataRegistry, oai_dc_reader , MetadataReader
View
6 deploy/fabfile.py
@@ -24,7 +24,7 @@ def deploy(branch, runTests=False):
'git checkout origin/%s' % branch]
updateLRInstall = [activateLrVirtualEnv,
changeToLRProjectDir,
- 'yes |pip uninstall LR',
+ 'yes | pip uninstall LR',
'yes | pip install .',
'yes | pip install uwsgi',
deactivate]
@@ -40,7 +40,7 @@ def deploy(branch, runTests=False):
'nosetests lr/tests/functional/test_slice.py']
run(';'.join(gitCheckout))
run(';'.join(updateLRInstall))
- run('killall -9 uwsgi;')
+ #run('killall -9 uwsgi;')
if runTests:
run(';'.join(testCommand))
- run(';'.join(startLRServer))
+ run('service learningregistry restart')
View
6 utils/upload.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
import urllib2,os,json,codecs
import ConfigParser
from random import choice
@@ -22,8 +22,8 @@
signer = Sign_0_21(privateKeyID=args.key ,publicKeyLocations=[args.key_location], passphrase=args.passphrase, gpgbin=args.gpgbin)
_config = ConfigParser.ConfigParser()
_config.read(args.config)
-root_path = _config.get("upload", "root_path")
-publish_url = _config.get("upload", "publish_url")
+#root_path = _config.get("upload", "root_path")
+#publish_url = _config.get("upload", "publish_url")
lr_test_data = args.lr_test_data.lower() in ["true", "t", "yes" "1"]
if args.publish_url != None:

0 comments on commit 8808861

Please sign in to comment.