Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

adding supported for related param in slice

gives us the ability to slice on related object id - e.g., ASN id for
standards alignment
  • Loading branch information...
commit 4da483c731d7b86c1d11c5bdbabdc2dc58a3f846 1 parent e349dc0
@joehobson joehobson authored
View
99 LR/lr/controllers/slice.py
@@ -10,6 +10,7 @@
from lr.lib.oaipmherrors import *
import types
from lr.lib import resumption_token
+import itertools
log = logging.getLogger(__name__)
@@ -17,6 +18,7 @@
START_DATE = 'from'
IDENTITY = 'identity'
ANY_TAGS = 'any_tags'
+RELATED = 'related'
#FULL_DOCS = 'full_docs'
IDS_ONLY = 'ids_only'
CALLBACK = 'callback'
@@ -108,6 +110,7 @@ def _set_boolean_param(paramKey, setifempty=True):
if _set_string_param(START_DATE) : param_count += 1
if _set_string_param(IDENTITY) : param_count += 1
if _set_string_param(ANY_TAGS) : param_count += 1
+ if _set_string_param(RELATED) : param_count += 1
_set_string_param(END_DATE)
_set_boolean_param(IDS_ONLY)
_set_string_param(CALLBACK, False)
@@ -128,7 +131,7 @@ def _set_boolean_param(paramKey, setifempty=True):
def _get_view(self,view_name = '_design/learningregistry-slice/_view/docs',keys=[], include_docs = False, resumptionToken=None, limit=None):
db_url = '/'.join([appConfig['couchdb.url'],appConfig['couchdb.db.resourcedata']])
- opts = {"stale": appConfig['couchdb.stale.flag'], "reduce": False }
+ opts = {"stale": "ok", "reduce": False }
if include_docs:
opts["include_docs"] = True
@@ -150,14 +153,9 @@ def _get_view(self,view_name = '_design/learningregistry-slice/_view/docs',keys=
return view
def _get_view_total(self,view_name = '_design/learningregistry-slice/_view/docs',keys=[], resumptionToken=None):
-
- if resumptionToken and "maxResults" in resumptionToken and resumptionToken["maxResults"] != None :
- return resumptionToken["maxResults"];
-
-
db_url = '/'.join([appConfig['couchdb.url'],appConfig['couchdb.db.resourcedata']])
- opts = {"stale": appConfig['couchdb.stale.flag'], "reduce": True, "group": True }
+ opts = {"stale": "ok", "reduce": True, "group": True }
if self.enable_flow_control and resumptionToken != None:
opts["keys"] = resumptionToken["keys"]
@@ -173,8 +171,6 @@ def _get_view_total(self,view_name = '_design/learningregistry-slice/_view/docs'
for row in view:
if "value" in row:
totalDocs += row["value"]
-
- #resumptionToken["maxResults"] = totalDocs;
return totalDocs
def _get_keys(self, params):
@@ -188,72 +184,50 @@ def _get_keys(self, params):
dates = [params[START_DATE]]
identity = params[IDENTITY].lower()
any_tags = params[ANY_TAGS].lower()
+ related = params[RELATED].lower()
param_count = params['param_count']
if any_tags != "" :
- any_tag_list = any_tags.split(",")
- wrapped_any_tag_list = []
- for tag in any_tag_list:
- try:
- #tag = "{\"tag\":\""+tag+"\"}"
- tag = {"tag":tag}
- wrapped_any_tag_list.append(tag)
- print("wrapped tag: " + str(tag))
- except:
- print("failed to wrap tag: " + str(tag))
- pass
- any_tag_list = wrapped_any_tag_list
+ try:
+ any_tags = [{"tag":t} for t in any_tags.split(",")]
+ for t in any_tags:
+ print("wrapped tag: " + str(t))
+ except:
+ print("failed to wrap tag: " + str(tag))
+ pass
if(identity != ""):
try:
- #identity = "{\"tag\":\""+identity+"\"}"
- identity = {"id":identity}
+ identity = [{"id":identity}]
print("wrapped identity: " + str(identity))
except:
pass
-
- wrapped_dates = []
- for date in dates:
+ if related != "":
try:
- #date = "{\"tag\":\""+date+"\"}"
- date = {"date":date}
- wrapped_dates.append(date)
- print("wrapped date: " + str(date))
+ related = [{"related":r} for r in related.split(",")]
+ for r in related:
+ print("wrapped related: " + str(r))
except:
- print("failed to wrap date: " + str(date))
pass
- dates = wrapped_dates
- if param_count == 1:
- if len(dates)>0 :
- for date in dates :
- keys.append(date)
- elif identity != "" :
- keys.append(identity)
- elif any_tags != "" :
- for tag in any_tag_list:
- keys.append(tag)
- elif param_count == 2:
- if len(dates) == 0 :
- for tag in any_tag_list:
- keys.append([identity, tag])
- elif identity == "" :
- for tag in any_tag_list:
- for date in dates:
- log.debug("slicegotdateandtag: " + str([date, tag]))
- keys.append([date, tag])
- elif any_tags == "" :
- for date in dates:
- keys.append([date, identity])
- elif param_count == 3:
- for tag in any_tag_list:
- for date in dates:
- keys.append([date, identity, tag])
-
+ try:
+ dates = [{"date":d} for d in dates]
+ for d in dates:
+ print("wrapped date: " + str(d))
+ except:
+ print("failed to wrap dates: " + str(dates))
+ pass
+
+ sources = [dates, any_tags, related, identity]
+ sources = [s for s in sources if (s != "" and len(s) > 0)]
+
+ if len(sources) > 1:
+ keys = list(apply(itertools.product, sources))
+ else:
+ keys = sources[0]
+
print("final slice keys: " + str(keys))
return keys
-
-
def _get_dates(self, params):
cur = datetime.strptime(params[START_DATE],"%Y-%m-%d")
@@ -274,7 +248,6 @@ def format_data(self,keys_only,docs, keys, forceUnique, current_rt=None):
prefix = '{"documents":[\n'
num_sent = 0
doc_count = 0
- update_resumption_max_results = current_rt and "maxResults" in current_rt and current_rt["maxResults"] != None
if docs is not None:
for row in docs:
doc_count += 1
@@ -294,8 +267,6 @@ def format_data(self,keys_only,docs, keys, forceUnique, current_rt=None):
prefix = ",\n"
else:
log.debug("{0} skipping: alreadySent {1} / forceUnique {2}".format(doc_count, repr(alreadySent), forceUnique))
- if update_resumption_max_results:
- current_rt["maxResults"] = current_rt["maxResults"] - 1
if doc_count == 0:
yield prefix
@@ -310,7 +281,7 @@ def format_data(self,keys_only,docs, keys, forceUnique, current_rt=None):
offset = 0
if offset+doc_count < maxResults:
- rt = ''' "resumption_token":"{0}", '''.format(resumption_token.get_offset_token(self.service_id, offset=offset+doc_count, keys=keys, maxResults=maxResults))
+ rt = ''' "resumption_token":"{0}", '''.format(resumption_token.get_offset_token(self.service_id, offset=offset+doc_count, keys=keys))
View
45 couchdb/resource_data/apps/learningregistry-slice/views/docs/map.js
@@ -2,6 +2,15 @@ function(doc) {
if (doc.doc_type != "resource_data" || !doc.node_timestamp) return;
+ //only emitting related as a single key so we don't kill this view
+ if(doc.resource_data && doc.resource_data.activity && doc.resource_data.activity.related) {
+ for each(var related in doc.resource_data.activity.related) {
+ if(related.id) {
+ emit({'related': related.id.toLowerCase() }, null);
+ }
+ }
+ }
+
var date_stamp = doc.node_timestamp;
date_stamp = date_stamp.substring(0,10);
var identities = new Array();
@@ -13,38 +22,14 @@ function(doc) {
return false;
}
-
//grab all the identities in identity or submitter/curator/owner/signer (depending on version)
- //if any identities are identical, ignore redundant ones.
if(doc.identity) {
- if(doc.identity.submitter) {
- identities.push(doc.identity.submitter.toLowerCase());
- }
- if(doc.identity.curator) {
- var curator = doc.identity.curator.toLowerCase();
- if(!arrayContains(identities,curator)) {
- identities.push(curator);
- }
- }
- if(doc.identity.owner) {
- var owner = doc.identity.owner.toLowerCase();
- if(!arrayContains(identities,owner)) {
- identities.push(owner);
- }
- }
- if(doc.identity.signer) {
- var signer = doc.identity.signer.toLowerCase();
- if(!arrayContains(identities,signer)) {
- identities.push(signer);
- }
- }
+ if(doc.identity.submitter) identities.push(doc.identity.submitter.toLowerCase());
+ if(doc.identity.curator) identities.push(doc.identity.curator.toLowerCase());
+ if(doc.identity.owner) identities.push(doc.identity.owner.toLowerCase());
+ if(doc.identity.signer) identities.push(doc.identity.signer.toLowerCase());
}
- if(doc.submitter) {
- var submitter = doc.submitter.toLowerCase();
- if(!arrayContains(identities,submitter)) {
- identities.push(submitter);
- }
- }
+ if(doc.submitter) identities.push(doc.submitter.toLowerCase());
//build identities indices
@@ -89,4 +74,6 @@ function(doc) {
if(doc.resource_data_type) emitaAllKeywordIndices(doc.resource_data_type.toLowerCase());
+
+
}
Please sign in to comment.
Something went wrong with that request. Please try again.