Skip to content
This repository has been archived by the owner on May 13, 2022. It is now read-only.

Commit

Permalink
adding supported for related param in slice
Browse files Browse the repository at this point in the history 
gives us the ability to slice on related object id - e.g., ASN id for
standards alignment
  • Loading branch information
@joehobson
joehobson committed Mar 14, 2012
1 parent e349dc0 commit 4da483c
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 93 deletions.
99 changes: 35 additions & 64 deletions LR/lr/controllers/slice.py
View file
Expand Up @@ -10,13 +10,15 @@
from lr.lib.oaipmherrors import * from lr.lib.oaipmherrors import *
import types import types
from lr.lib import resumption_token from lr.lib import resumption_token
import itertools


log = logging.getLogger(__name__) log = logging.getLogger(__name__)


END_DATE = 'until' END_DATE = 'until'
START_DATE = 'from' START_DATE = 'from'
IDENTITY = 'identity' IDENTITY = 'identity'
ANY_TAGS = 'any_tags' ANY_TAGS = 'any_tags'
RELATED = 'related'
#FULL_DOCS = 'full_docs' #FULL_DOCS = 'full_docs'
IDS_ONLY = 'ids_only' IDS_ONLY = 'ids_only'
CALLBACK = 'callback' CALLBACK = 'callback'
Expand Down Expand Up @@ -108,6 +110,7 @@ def _set_boolean_param(paramKey, setifempty=True):
if _set_string_param(START_DATE) : param_count += 1 if _set_string_param(START_DATE) : param_count += 1
if _set_string_param(IDENTITY) : param_count += 1 if _set_string_param(IDENTITY) : param_count += 1
if _set_string_param(ANY_TAGS) : param_count += 1 if _set_string_param(ANY_TAGS) : param_count += 1
if _set_string_param(RELATED) : param_count += 1
_set_string_param(END_DATE) _set_string_param(END_DATE)
_set_boolean_param(IDS_ONLY) _set_boolean_param(IDS_ONLY)
_set_string_param(CALLBACK, False) _set_string_param(CALLBACK, False)
Expand All @@ -128,7 +131,7 @@ def _set_boolean_param(paramKey, setifempty=True):
def _get_view(self,view_name = '_design/learningregistry-slice/_view/docs',keys=[], include_docs = False, resumptionToken=None, limit=None): def _get_view(self,view_name = '_design/learningregistry-slice/_view/docs',keys=[], include_docs = False, resumptionToken=None, limit=None):
db_url = '/'.join([appConfig['couchdb.url'],appConfig['couchdb.db.resourcedata']]) db_url = '/'.join([appConfig['couchdb.url'],appConfig['couchdb.db.resourcedata']])


opts = {"stale": appConfig['couchdb.stale.flag'], "reduce": False } opts = {"stale": "ok", "reduce": False }


if include_docs: if include_docs:
opts["include_docs"] = True opts["include_docs"] = True
Expand All @@ -150,14 +153,9 @@ def _get_view(self,view_name = '_design/learningregistry-slice/_view/docs',keys=
return view return view


def _get_view_total(self,view_name = '_design/learningregistry-slice/_view/docs',keys=[], resumptionToken=None): def _get_view_total(self,view_name = '_design/learningregistry-slice/_view/docs',keys=[], resumptionToken=None):

if resumptionToken and "maxResults" in resumptionToken and resumptionToken["maxResults"] != None :
return resumptionToken["maxResults"];


db_url = '/'.join([appConfig['couchdb.url'],appConfig['couchdb.db.resourcedata']]) db_url = '/'.join([appConfig['couchdb.url'],appConfig['couchdb.db.resourcedata']])


opts = {"stale": appConfig['couchdb.stale.flag'], "reduce": True, "group": True } opts = {"stale": "ok", "reduce": True, "group": True }


if self.enable_flow_control and resumptionToken != None: if self.enable_flow_control and resumptionToken != None:
opts["keys"] = resumptionToken["keys"] opts["keys"] = resumptionToken["keys"]
Expand All @@ -173,8 +171,6 @@ def _get_view_total(self,view_name = '_design/learningregistry-slice/_view/docs'
for row in view: for row in view:
if "value" in row: if "value" in row:
totalDocs += row["value"] totalDocs += row["value"]

#resumptionToken["maxResults"] = totalDocs;
return totalDocs return totalDocs


def _get_keys(self, params): def _get_keys(self, params):
Expand All @@ -188,72 +184,50 @@ def _get_keys(self, params):
dates = [params[START_DATE]] dates = [params[START_DATE]]
identity = params[IDENTITY].lower() identity = params[IDENTITY].lower()
any_tags = params[ANY_TAGS].lower() any_tags = params[ANY_TAGS].lower()
related = params[RELATED].lower()


param_count = params['param_count'] param_count = params['param_count']


if any_tags != "" : if any_tags != "" :
any_tag_list = any_tags.split(",") try:
wrapped_any_tag_list = [] any_tags = [{"tag":t} for t in any_tags.split(",")]
for tag in any_tag_list: for t in any_tags:
try: print("wrapped tag: " + str(t))
#tag = "{\"tag\":\""+tag+"\"}" except:
tag = {"tag":tag} print("failed to wrap tag: " + str(tag))
wrapped_any_tag_list.append(tag) pass
print("wrapped tag: " + str(tag))
except:
print("failed to wrap tag: " + str(tag))
pass
any_tag_list = wrapped_any_tag_list
if(identity != ""): if(identity != ""):
try: try:
#identity = "{\"tag\":\""+identity+"\"}" identity = [{"id":identity}]
identity = {"id":identity}
print("wrapped identity: " + str(identity)) print("wrapped identity: " + str(identity))
except: except:
pass pass

if related != "":
wrapped_dates = []
for date in dates:
try: try:
#date = "{\"tag\":\""+date+"\"}" related = [{"related":r} for r in related.split(",")]
date = {"date":date} for r in related:
wrapped_dates.append(date) print("wrapped related: " + str(r))
print("wrapped date: " + str(date))
except: except:
print("failed to wrap date: " + str(date))
pass pass
dates = wrapped_dates


if param_count == 1: try:
if len(dates)>0 : dates = [{"date":d} for d in dates]
for date in dates : for d in dates:
keys.append(date) print("wrapped date: " + str(d))
elif identity != "" : except:
keys.append(identity) print("failed to wrap dates: " + str(dates))
elif any_tags != "" : pass
for tag in any_tag_list:
keys.append(tag) sources = [dates, any_tags, related, identity]
elif param_count == 2: sources = [s for s in sources if (s != "" and len(s) > 0)]
if len(dates) == 0 :
for tag in any_tag_list: if len(sources) > 1:
keys.append([identity, tag]) keys = list(apply(itertools.product, sources))
elif identity == "" : else:
for tag in any_tag_list: keys = sources[0]
for date in dates:
log.debug("slicegotdateandtag: " + str([date, tag]))
keys.append([date, tag])
elif any_tags == "" :
for date in dates:
keys.append([date, identity])
elif param_count == 3:
for tag in any_tag_list:
for date in dates:
keys.append([date, identity, tag])

print("final slice keys: " + str(keys)) print("final slice keys: " + str(keys))
return keys return keys




def _get_dates(self, params): def _get_dates(self, params):
cur = datetime.strptime(params[START_DATE],"%Y-%m-%d") cur = datetime.strptime(params[START_DATE],"%Y-%m-%d")
Expand All @@ -274,7 +248,6 @@ def format_data(self,keys_only,docs, keys, forceUnique, current_rt=None):
prefix = '{"documents":[\n' prefix = '{"documents":[\n'
num_sent = 0 num_sent = 0
doc_count = 0 doc_count = 0
update_resumption_max_results = current_rt and "maxResults" in current_rt and current_rt["maxResults"] != None
if docs is not None: if docs is not None:
for row in docs: for row in docs:
doc_count += 1 doc_count += 1
Expand All @@ -294,8 +267,6 @@ def format_data(self,keys_only,docs, keys, forceUnique, current_rt=None):
prefix = ",\n" prefix = ",\n"
else: else:
log.debug("{0} skipping: alreadySent {1} / forceUnique {2}".format(doc_count, repr(alreadySent), forceUnique)) log.debug("{0} skipping: alreadySent {1} / forceUnique {2}".format(doc_count, repr(alreadySent), forceUnique))
if update_resumption_max_results:
current_rt["maxResults"] = current_rt["maxResults"] - 1


if doc_count == 0: if doc_count == 0:
yield prefix yield prefix
Expand All @@ -310,7 +281,7 @@ def format_data(self,keys_only,docs, keys, forceUnique, current_rt=None):
offset = 0 offset = 0


if offset+doc_count < maxResults: if offset+doc_count < maxResults:
rt = ''' "resumption_token":"{0}", '''.format(resumption_token.get_offset_token(self.service_id, offset=offset+doc_count, keys=keys, maxResults=maxResults)) rt = ''' "resumption_token":"{0}", '''.format(resumption_token.get_offset_token(self.service_id, offset=offset+doc_count, keys=keys))






Expand Down
45 changes: 16 additions & 29 deletions couchdb/resource_data/apps/learningregistry-slice/views/docs/map.js
View file
Expand Up @@ -2,6 +2,15 @@ function(doc) {


if (doc.doc_type != "resource_data" || !doc.node_timestamp) return; if (doc.doc_type != "resource_data" || !doc.node_timestamp) return;


//only emitting related as a single key so we don't kill this view
if(doc.resource_data && doc.resource_data.activity && doc.resource_data.activity.related) {
for each(var related in doc.resource_data.activity.related) {
if(related.id) {
emit({'related': related.id.toLowerCase() }, null);
}
}
}

var date_stamp = doc.node_timestamp; var date_stamp = doc.node_timestamp;
date_stamp = date_stamp.substring(0,10); date_stamp = date_stamp.substring(0,10);
var identities = new Array(); var identities = new Array();
Expand All @@ -13,38 +22,14 @@ function(doc) {
return false; return false;
} }



//grab all the identities in identity or submitter/curator/owner/signer (depending on version) //grab all the identities in identity or submitter/curator/owner/signer (depending on version)
//if any identities are identical, ignore redundant ones.
if(doc.identity) { if(doc.identity) {
if(doc.identity.submitter) { if(doc.identity.submitter) identities.push(doc.identity.submitter.toLowerCase());
identities.push(doc.identity.submitter.toLowerCase()); if(doc.identity.curator) identities.push(doc.identity.curator.toLowerCase());
} if(doc.identity.owner) identities.push(doc.identity.owner.toLowerCase());
if(doc.identity.curator) { if(doc.identity.signer) identities.push(doc.identity.signer.toLowerCase());
var curator = doc.identity.curator.toLowerCase();
if(!arrayContains(identities,curator)) {
identities.push(curator);
}
}
if(doc.identity.owner) {
var owner = doc.identity.owner.toLowerCase();
if(!arrayContains(identities,owner)) {
identities.push(owner);
}
}
if(doc.identity.signer) {
var signer = doc.identity.signer.toLowerCase();
if(!arrayContains(identities,signer)) {
identities.push(signer);
}
}
} }
if(doc.submitter) { if(doc.submitter) identities.push(doc.submitter.toLowerCase());
var submitter = doc.submitter.toLowerCase();
if(!arrayContains(identities,submitter)) {
identities.push(submitter);
}
}




//build identities indices //build identities indices
Expand Down Expand Up @@ -89,4 +74,6 @@ function(doc) {


if(doc.resource_data_type) emitaAllKeywordIndices(doc.resource_data_type.toLowerCase()); if(doc.resource_data_type) emitaAllKeywordIndices(doc.resource_data_type.toLowerCase());




} }

0 comments on commit 4da483c

Please sign in to comment.