Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

adding supported for related param in slice

gives us the ability to slice on related object id - e.g., ASN id for
standards alignment
  • Loading branch information...
commit 4da483c731d7b86c1d11c5bdbabdc2dc58a3f846 1 parent e349dc0
joe hobson authored March 13, 2012
99  LR/lr/controllers/slice.py
@@ -10,6 +10,7 @@
10 10
 from lr.lib.oaipmherrors import *
11 11
 import types
12 12
 from lr.lib import resumption_token
  13
+import itertools
13 14
 
14 15
 log = logging.getLogger(__name__)
15 16
 
@@ -17,6 +18,7 @@
17 18
 START_DATE = 'from'
18 19
 IDENTITY = 'identity'
19 20
 ANY_TAGS = 'any_tags'
  21
+RELATED = 'related'
20 22
 #FULL_DOCS = 'full_docs'
21 23
 IDS_ONLY = 'ids_only'
22 24
 CALLBACK = 'callback'
@@ -108,6 +110,7 @@ def _set_boolean_param(paramKey, setifempty=True):
108 110
         if _set_string_param(START_DATE) : param_count += 1
109 111
         if _set_string_param(IDENTITY) : param_count += 1
110 112
         if _set_string_param(ANY_TAGS) : param_count += 1
  113
+        if _set_string_param(RELATED) : param_count += 1
111 114
         _set_string_param(END_DATE)
112 115
         _set_boolean_param(IDS_ONLY)
113 116
         _set_string_param(CALLBACK, False)
@@ -128,7 +131,7 @@ def _set_boolean_param(paramKey, setifempty=True):
128 131
     def _get_view(self,view_name = '_design/learningregistry-slice/_view/docs',keys=[], include_docs = False, resumptionToken=None, limit=None):
129 132
         db_url = '/'.join([appConfig['couchdb.url'],appConfig['couchdb.db.resourcedata']])
130 133
         
131  
-        opts = {"stale": appConfig['couchdb.stale.flag'], "reduce": False }
  134
+        opts = {"stale": "ok", "reduce": False }
132 135
         
133 136
         if include_docs:
134 137
             opts["include_docs"] = True
@@ -150,14 +153,9 @@ def _get_view(self,view_name = '_design/learningregistry-slice/_view/docs',keys=
150 153
         return view
151 154
     
152 155
     def _get_view_total(self,view_name = '_design/learningregistry-slice/_view/docs',keys=[], resumptionToken=None):
153  
-        
154  
-        if resumptionToken and "maxResults" in resumptionToken and resumptionToken["maxResults"] != None :
155  
-            return resumptionToken["maxResults"];
156  
-            
157  
-        
158 156
         db_url = '/'.join([appConfig['couchdb.url'],appConfig['couchdb.db.resourcedata']])
159 157
         
160  
-        opts = {"stale": appConfig['couchdb.stale.flag'], "reduce": True, "group": True }
  158
+        opts = {"stale": "ok", "reduce": True, "group": True }
161 159
         
162 160
         if self.enable_flow_control and resumptionToken != None:
163 161
             opts["keys"] = resumptionToken["keys"]
@@ -173,8 +171,6 @@ def _get_view_total(self,view_name = '_design/learningregistry-slice/_view/docs'
173 171
         for row in view:
174 172
             if "value" in row:
175 173
                 totalDocs += row["value"]
176  
-        
177  
-        #resumptionToken["maxResults"] = totalDocs;
178 174
         return totalDocs
179 175
     
180 176
     def _get_keys(self, params):
@@ -188,72 +184,50 @@ def _get_keys(self, params):
188 184
                 dates = [params[START_DATE]]
189 185
         identity = params[IDENTITY].lower()
190 186
         any_tags = params[ANY_TAGS].lower()
  187
+        related = params[RELATED].lower()
191 188
         
192 189
         param_count = params['param_count']
193 190
         
194 191
         if any_tags != "" :
195  
-            any_tag_list = any_tags.split(",")
196  
-            wrapped_any_tag_list = []
197  
-            for tag in any_tag_list:
198  
-                try:
199  
-                    #tag = "{\"tag\":\""+tag+"\"}"
200  
-                    tag = {"tag":tag}
201  
-                    wrapped_any_tag_list.append(tag)
202  
-                    print("wrapped tag: " + str(tag))
203  
-                except:
204  
-                    print("failed to wrap tag: " + str(tag))
205  
-                    pass
206  
-            any_tag_list = wrapped_any_tag_list
  192
+            try:
  193
+                any_tags = [{"tag":t} for t in any_tags.split(",")]
  194
+                for t in any_tags:
  195
+                    print("wrapped tag: " + str(t))
  196
+            except:
  197
+                print("failed to wrap tag: " + str(tag))
  198
+                pass
207 199
         if(identity != ""):
208 200
             try:
209  
-                #identity = "{\"tag\":\""+identity+"\"}"
210  
-                identity = {"id":identity}
  201
+                identity = [{"id":identity}]
211 202
                 print("wrapped identity: " + str(identity))
212 203
             except:
213 204
             	pass
214  
-        
215  
-        wrapped_dates = []
216  
-        for date in dates:
  205
+        if related != "":
217 206
             try:
218  
-                #date = "{\"tag\":\""+date+"\"}"
219  
-                date = {"date":date}
220  
-                wrapped_dates.append(date)
221  
-                print("wrapped date: " + str(date))
  207
+                related = [{"related":r} for r in related.split(",")]
  208
+                for r in related:
  209
+                    print("wrapped related: " + str(r))
222 210
             except:
223  
-                print("failed to wrap date: " + str(date))
224 211
                 pass
225  
-        dates = wrapped_dates
226 212
         
227  
-        if param_count == 1:
228  
-            if len(dates)>0 :
229  
-                for date in dates :
230  
-                    keys.append(date)
231  
-            elif identity != "" :
232  
-                keys.append(identity)
233  
-            elif any_tags != "" :
234  
-                for tag in any_tag_list:
235  
-                    keys.append(tag)
236  
-        elif param_count == 2:
237  
-            if len(dates) == 0 :
238  
-                for tag in any_tag_list:
239  
-                    keys.append([identity, tag])
240  
-            elif identity == "" :
241  
-                for tag in any_tag_list:
242  
-                    for date in dates:
243  
-                        log.debug("slicegotdateandtag: " + str([date, tag]))
244  
-                        keys.append([date, tag])
245  
-            elif any_tags == "" :
246  
-                for date in dates:
247  
-                    keys.append([date, identity])
248  
-        elif param_count == 3:
249  
-            for tag in any_tag_list:
250  
-                for date in dates:
251  
-                    keys.append([date, identity, tag])
252  
-         
  213
+        try:
  214
+            dates = [{"date":d} for d in dates]
  215
+            for d in dates:    
  216
+                print("wrapped date: " + str(d))
  217
+        except:
  218
+            print("failed to wrap dates: " + str(dates))
  219
+            pass
  220
+        
  221
+        sources = [dates, any_tags, related, identity]
  222
+        sources = [s for s in sources if (s != "" and len(s) > 0)]
  223
+        
  224
+        if len(sources) > 1:
  225
+            keys = list(apply(itertools.product, sources))
  226
+        else:
  227
+            keys = sources[0]
  228
+            
253 229
         print("final slice keys: " + str(keys))
254 230
         return keys
255  
-    
256  
-    
257 231
         
258 232
     def _get_dates(self, params):
259 233
         cur = datetime.strptime(params[START_DATE],"%Y-%m-%d")
@@ -274,7 +248,6 @@ def format_data(self,keys_only,docs, keys, forceUnique, current_rt=None):
274 248
         prefix = '{"documents":[\n'
275 249
         num_sent = 0
276 250
         doc_count = 0
277  
-        update_resumption_max_results = current_rt and "maxResults" in current_rt and current_rt["maxResults"] != None
278 251
         if docs is not None:
279 252
             for row in docs:
280 253
                 doc_count += 1
@@ -294,8 +267,6 @@ def format_data(self,keys_only,docs, keys, forceUnique, current_rt=None):
294 267
                     prefix = ",\n"
295 268
                 else:
296 269
                     log.debug("{0} skipping: alreadySent {1} / forceUnique {2}".format(doc_count, repr(alreadySent), forceUnique))
297  
-                    if update_resumption_max_results:
298  
-                        current_rt["maxResults"] = current_rt["maxResults"] - 1
299 270
         
300 271
         if doc_count == 0:
301 272
             yield prefix
@@ -310,7 +281,7 @@ def format_data(self,keys_only,docs, keys, forceUnique, current_rt=None):
310 281
                 offset = 0
311 282
                 
312 283
             if offset+doc_count < maxResults:
313  
-                rt = ''' "resumption_token":"{0}", '''.format(resumption_token.get_offset_token(self.service_id, offset=offset+doc_count, keys=keys, maxResults=maxResults))
  284
+                rt = ''' "resumption_token":"{0}", '''.format(resumption_token.get_offset_token(self.service_id, offset=offset+doc_count, keys=keys))
314 285
 
315 286
         
316 287
 
45  couchdb/resource_data/apps/learningregistry-slice/views/docs/map.js
@@ -2,6 +2,15 @@ function(doc) {
2 2
 
3 3
 	if (doc.doc_type != "resource_data" || !doc.node_timestamp) return;
4 4
 	
  5
+	//only emitting related as a single key so we don't kill this view
  6
+	if(doc.resource_data && doc.resource_data.activity && doc.resource_data.activity.related) {
  7
+	    for each(var related in doc.resource_data.activity.related) {
  8
+		if(related.id) {
  9
+	            emit({'related': related.id.toLowerCase() }, null);
  10
+		}
  11
+	    }
  12
+	}
  13
+
5 14
 	var date_stamp = doc.node_timestamp;
6 15
 	date_stamp = date_stamp.substring(0,10);
7 16
 	var identities = new Array();
@@ -13,38 +22,14 @@ function(doc) {
13 22
 		return false;
14 23
 	}	
15 24
 	
16  
-	
17 25
 	//grab all the identities in identity or submitter/curator/owner/signer (depending on version)
18  
-	//if any identities are identical, ignore redundant ones.
19 26
 	if(doc.identity) {
20  
-		if(doc.identity.submitter) {
21  
-			identities.push(doc.identity.submitter.toLowerCase());
22  
-		}
23  
-		if(doc.identity.curator) {
24  
-			var curator = doc.identity.curator.toLowerCase();
25  
-			if(!arrayContains(identities,curator)) {
26  
-				identities.push(curator);
27  
-			}
28  
-		}
29  
-		if(doc.identity.owner) {
30  
-			var owner = doc.identity.owner.toLowerCase();
31  
-			if(!arrayContains(identities,owner)) {
32  
-				identities.push(owner);
33  
-			}
34  
-		}
35  
-		if(doc.identity.signer) {
36  
-			var signer = doc.identity.signer.toLowerCase();
37  
-			if(!arrayContains(identities,signer)) {
38  
-				identities.push(signer);
39  
-			}
40  
-		}
  27
+		if(doc.identity.submitter) identities.push(doc.identity.submitter.toLowerCase());
  28
+		if(doc.identity.curator) identities.push(doc.identity.curator.toLowerCase());
  29
+		if(doc.identity.owner) identities.push(doc.identity.owner.toLowerCase());
  30
+		if(doc.identity.signer) identities.push(doc.identity.signer.toLowerCase());
41 31
 	}
42  
-	if(doc.submitter) {
43  
-		var submitter = doc.submitter.toLowerCase();
44  
-		if(!arrayContains(identities,submitter)) {
45  
-			identities.push(submitter);
46  
-		}
47  
-	} 
  32
+	if(doc.submitter) identities.push(doc.submitter.toLowerCase());
48 33
 	
49 34
 
50 35
 	//build identities indices
@@ -89,4 +74,6 @@ function(doc) {
89 74
 	
90 75
 	if(doc.resource_data_type) emitaAllKeywordIndices(doc.resource_data_type.toLowerCase());
91 76
 	
  77
+	
  78
+    
92 79
 }

0 notes on commit 4da483c

Please sign in to comment.
Something went wrong with that request. Please try again.