Merge branch 'master' into updatePylonsInstall

LR/development.ini.orig

@@ -69,6 +69,7 @@ couchdb.db.resourcesview = _design/learningregistry-resources/_view/docs
 couchdb.db.resourcecount = _design/learningregistry-resources/_view/count
 couchdb.threshold.distributes = 1000
 couchdb.threshold.viewupdate = 100
+couchdb.stale.flag = update_after
 lr.distribute.url = http://localhost/distribute
 lr.obtain.docid = access:Basic Obtain service
 lr.harvest.docid = access:Basic Harvest service
@@ -82,16 +83,20 @@ lr.services.docid = administrative:Network Node Services service
 lr.policy.docid = administrative:Resource Distribution Network Policy service
 lr.status.docid = administrative:Network Node Status service
 lr.distribute.docid = distribute:Resource Data Distribution service
-models_spec_dir = %(here)s/data/models
-spec.models.node_description = %(here)s/data/LR_Tech_Spec_V_0_23/models/node_description
-spec.models.resource_data = %(here)s/data/LR_Tech_Spec_V_0_23/models/resource_data
-spec.models.network_policy_description = %(here)s/data/LR_Tech_Spec_V_0_23/models/policy_description
-spec.models.status_description = %(here)s/data/LR_Tech_Spec_V_0_23/models/status_description
-spec.models.filter_description = %(here)s/data/LR_Tech_Spec_V_0_23/models/filter_description
-spec.models.node_connectivity_description = %(here)s/data/LR_Tech_Spec_V_0_23/models/connection_description
-spec.models.community_description = %(here)s/data/LR_Tech_Spec_V_0_23/models/community_description
-spec.models.network_description = %(here)s/data/LR_Tech_Spec_V_0_23/models/network_description
-spec.models.node_service_description = %(here)s/data/LR_Tech_Spec_V_0_23/models/service_description
+specs_dir = %(here)s/data/specs
+spec.models.node_description =%(specs_dir)s/v_0_23/models/node_description
+
+spec.models.resource_data =%(specs_dir)s/v_0_23/models/resource_data_description,
+                                                %(specs_dir)s/v_0_21/models/resource_data_description,
+                                                %(specs_dir)s/v_0_17/models/resource_data_description
+
+spec.models.network_policy_description =%(specs_dir)s/v_0_23/models/policy_description
+spec.models.status_description =%(specs_dir)s/v_0_23/models/status_description
+spec.models.filter_description =%(specs_dir)s/v_0_23/models/filter_description
+spec.models.node_connectivity_description =%(specs_dir)s/v_0_23/models/connection_description
+spec.models.community_description =%(specs_dir)s/v_0_23/models/community_description
+spec.models.network_description =%(specs_dir)s/v_0_23/models/network_description
+spec.models.node_service_description =%(specs_dir)s/v_0_23/models/service_description
 
 
 

LR/lr/config/routing.py

@@ -29,6 +29,10 @@ def mapResource(config_key, member_name, collection_name):
                 map.connect("/"+collection_name,controller=collection_name,action='options',conditions=dict(method=['OPTIONS']))
                 if member_name == 'swordservice':
                     map.connect("/swordpub",controller='swordservice',action='create')
+
+                if member_name == 'distribute':
+                    map.connect("/destination", controller='distribute', action='destination',
+                                          conditions=dict(method='GET'))
                 log.info("Enabling service route for: {0} member: {1} collection: {2}".format(service_doc_id, member_name, collection_name))
             else:
                 log.info("Service route for {0} is disabled".format(service_doc_id))

LR/lr/controllers/harvest.py

@@ -13,7 +13,6 @@
 log = logging.getLogger(__name__)
 import ast
 import string
-
 from lr.model import LRNode as sourceLRNode, \
             NodeServiceModel, ResourceDataModel, LRNodeModel, defaultCouchServer, appConfig
 BASIC_HARVEST_SERVICE_DOC = appConfig['lr.harvest.docid']

LR/lr/controllers/obtain.py

@@ -33,7 +33,7 @@ def get_view(self,view_name = '_design/learningregistry-resources/_view/docs',ke
         args = {}
         if len(keys) > 0:
             args['keys'] = keys
-        args['stale'] = 'ok'
+        args['stale'] = appConfig['couchdb.stale.flag']
         if self.limit is not None:
             args['limit'] = self.limit
         args['include_docs'] = include_docs

LR/lr/controllers/slice.py

@@ -128,7 +128,7 @@ def _set_boolean_param(paramKey, setifempty=True):
     def _get_view(self,view_name = '_design/learningregistry-slice/_view/docs',keys=[], include_docs = False, resumptionToken=None, limit=None):
         db_url = '/'.join([appConfig['couchdb.url'],appConfig['couchdb.db.resourcedata']])
 
-        opts = {"stale": "ok", "reduce": False }
+        opts = {"stale": appConfig['couchdb.stale.flag'], "reduce": False }
 
         if include_docs:
             opts["include_docs"] = True
@@ -150,9 +150,14 @@ def _get_view(self,view_name = '_design/learningregistry-slice/_view/docs',keys=
         return view
 
     def _get_view_total(self,view_name = '_design/learningregistry-slice/_view/docs',keys=[], resumptionToken=None):
+
+        if resumptionToken and "maxResults" in resumptionToken and resumptionToken["maxResults"] != None :
+            return resumptionToken["maxResults"];
+
+
         db_url = '/'.join([appConfig['couchdb.url'],appConfig['couchdb.db.resourcedata']])
 
-        opts = {"stale": "ok", "reduce": True, "group": True }
+        opts = {"stale": appConfig['couchdb.stale.flag'], "reduce": True, "group": True }
 
         if self.enable_flow_control and resumptionToken != None:
             opts["keys"] = resumptionToken["keys"]
@@ -168,6 +173,8 @@ def _get_view_total(self,view_name = '_design/learningregistry-slice/_view/docs'
         for row in view:
             if "value" in row:
                 totalDocs += row["value"]
+
+        #resumptionToken["maxResults"] = totalDocs;
         return totalDocs
 
     def _get_keys(self, params):
@@ -267,6 +274,7 @@ def format_data(self,keys_only,docs, keys, forceUnique, current_rt=None):
         prefix = '{"documents":[\n'
         num_sent = 0
         doc_count = 0
+        update_resumption_max_results = current_rt and "maxResults" in current_rt and current_rt["maxResults"] != None
         if docs is not None:
             for row in docs:
                 doc_count += 1
@@ -286,6 +294,8 @@ def format_data(self,keys_only,docs, keys, forceUnique, current_rt=None):
                     prefix = ",\n"
                 else:
                     log.debug("{0} skipping: alreadySent {1} / forceUnique {2}".format(doc_count, repr(alreadySent), forceUnique))
+                    if update_resumption_max_results:
+                        current_rt["maxResults"] = current_rt["maxResults"] - 1
 
         if doc_count == 0:
             yield prefix
@@ -300,11 +310,11 @@ def format_data(self,keys_only,docs, keys, forceUnique, current_rt=None):
                 offset = 0
 
             if offset+doc_count < maxResults:
-                rt = ''' "resumption_token":"{0}", '''.format(resumption_token.get_offset_token(self.service_id, offset=offset+doc_count, keys=keys))
+                rt = ''' "resumption_token":"{0}", '''.format(resumption_token.get_offset_token(self.service_id, offset=offset+doc_count, keys=keys, maxResults=maxResults))
 
 
 
-        yield '\n],'+rt+'"resultCount":'+str(num_sent) +'}'
+        yield '\n],'+rt+'"resultCount":'+str(maxResults) +'}'
 
 # if __name__ == '__main__':
 # param = {START_DATE: "2011-03-10", END_DATE: "2011-05-01", IDENTITY: "NSDL 2 LR Data Pump", 'search_key': 'Arithmetic'}

LR/lr/lib/harvest.py

@@ -23,7 +23,7 @@ def get_record(self,id):
           return None   
 
   def get_records_by_resource(self,resource_locator):
-    view_data = h.getView(database_url=self.db_url,view_name='_design/learningregistry-resource-location/_view/docs',method="POST",include_docs=True,keys=[resource_locator], stale='ok')
+    view_data = h.getView(database_url=self.db_url,view_name='_design/learningregistry-resource-location/_view/docs',method="POST",include_docs=True,keys=[resource_locator], stale=appConfig['couchdb.stale.flag'])
     for doc in view_data:
         yield doc["doc"]      
 
@@ -33,7 +33,7 @@ def list_records(self, from_date, until_date,resumption_token=None, limit=None):
   def list_metadata_formats(self):
      return [{'metadataFormat':{'metadataPrefix':'dc'}}]
   def earliestDate(self):
-    view = self.db.view('_design/learningregistry-by-date/_view/docs',limit=1,stale='ok')
+    view = self.db.view('_design/learningregistry-by-date/_view/docs',limit=1,stale=appConfig['couchdb.stale.flag'])
     if len(view.rows) > 0:
       return view.rows[0].key
     else:
@@ -42,7 +42,7 @@ def list_identifiers(self, from_date, until_date,resumption_token=None,limit=Non
     return self.getViewRows(False,until_date,from_date,limit,resumption_token)
   def getViewRows(self,includeDocs, untilDate,fromDate,limit=None,resumption_token=None):    
     params = {
-        'stale':'ok',
+        'stale':appConfig['couchdb.stale.flag'],
         'include_docs':includeDocs,
         'endkey':h.convertToISO8601Zformat(untilDate),
         'startkey':h.convertToISO8601Zformat(fromDate),

LR/lr/lib/model_parser.py

@@ -460,6 +460,14 @@ def _validateField(self, fieldName, value,  modelProp):
                 +fieldName+"' of type "+
                 modelProp[self._VALUE_TYPE]+"\n\n"+description)
 
+        #Check that required string are not empty.
+        if (self._IS_REQUIRED in modelProp.keys() and 
+            isOfSpecType(value, 'string') and 
+            len(value.strip()) == 0):
+                raise SpecValidationException(
+                    "Required value for '"+fieldName+"' cannot be an empty string\n\n"
+                    +description)
+
          # Check for matching defined value
         if (self._VALUE_DEFINED in modelProp.keys() and
             modelProp[self._VALUE_DEFINED] != value):
@@ -526,10 +534,14 @@ def _getModelInfo(self):
         self._extractData()
         return self._modelInfo
 
-
+    def _getDocVersion(self):
+        if 'doc_version' in self.modelInfo:
+            return self.modelInfo.doc_version.value
+        return None
 
     modelInfo = property(_getModelInfo, None, None, None)
     modelName = property(lambda self: self._modelName, None, None, None)
+    docVersion = property(_getDocVersion, None, None, None)
 
     def asJSON(self):
         """Transforms the parsed model to a valid JSON representation."""
@@ -610,28 +622,21 @@ def extractModelsFromSpec(specFile, destDir='./models/'):
 
     parser = OptionParser()
     parser.add_option("-f", "--file", dest="filepath", 
-                      help="The full path of the data model spec definition.",
-                      metavar="FILE")
+                      help="The full path of the data model spec definition.",  metavar="FILE")
 
-    parser.add_option("-s", "--string", dest="string",
-                      help="String representation of the data model spec definition")
+    parser.add_option("-s", "--string", dest="string",  help="String representation of the data model spec definition")
 
     parser.add_option("-j", "--json", dest="json", action = "store_true", 
-                      default=False,
-                      help="Show a json representation of data model spec.")
+                      default=False,  help="Show a json representation of data model spec.")
 
-    parser.add_option("-v", "--validate", dest="source",
-                      help="""Validates a JSON object against the spec data model
+    parser.add_option("-v", "--validate", dest="source",  help="""Validates a JSON object against the spec data model
                             The source JSON object can be a file or a string 
                             representation of the JSON object.
                             """
                         )
-    parser.add_option("-e", "--extract", dest="specFile",
-                      help="extracts data models from the spec file.",
-                      metavar="FILE")
+    parser.add_option("-e", "--extract", dest="specFile",  help="extracts data models from the spec file.",  metavar="FILE")
 
-    parser.add_option("-d", "--destination", dest="modelDestination",
-                      help="""Destination path to put data model file specs 
+    parser.add_option("-d", "--destination", dest="modelDestination",  help="""Destination path to put data model file specs 
                             that are extracted from the main spec.""")
 
 

LR/lr/lib/oaipmh.py

@@ -151,13 +151,13 @@ def list_opts(self, metadataPrefix, from_date=None, until_date=None):
 #        return map(lambda row: row["value"], view_data)
 
     def get_records_by_resource(self,resource_locator):
-        view_data = h.getView(database_url=self.db_url,view_name='_design/learningregistry-resource-location/_view/docs',method="POST", documentHandler=OAIPMHDocumentResolver(), include_docs=True,keys=[resource_locator], stale='ok')
+        view_data = h.getView(database_url=self.db_url,view_name='_design/learningregistry-resource-location/_view/docs',method="POST", documentHandler=OAIPMHDocumentResolver(), include_docs=True,keys=[resource_locator], stale=appConfig['couchdb.stale.flag'])
         for doc in view_data:
             yield doc  
 
     def list_identifiers_or_records(self,metadataPrefix,from_date=None, until_date=None, rt=None, fc_limit=None, serviceid=None, include_docs=False):
         '''Returns the list_records as a generator based upon OAI-PMH query'''
-        opts = { "stale": "ok" };
+        opts = { "stale": appConfig['couchdb.stale.flag'] };
         if include_docs:
             opts["include_docs"] = True
 
@@ -195,7 +195,7 @@ def format_docs(row):
 
     def list_metadata_formats(self, identity=None, by_doc_ID=False, verb="ListMetadataFormats"):
         try:
-            opts = { "stale": "ok" }
+            opts = { "stale": appConfig['couchdb.stale.flag'] }
             if identity != None:
                 opts["include_docs"] = "true"
 
@@ -243,7 +243,7 @@ def identify(self, database="node"):
             opts = {
                     "group": True,
                     "limit": 1,
-                    "stale": "ok"
+                    "stale": appConfig['couchdb.stale.flag']
                     }
 
             view_data = self.db.view('oai-pmh-identify-timestamp/docs', **opts)

LR/lr/lib/resumption_token.py

@@ -39,21 +39,23 @@ def get_payload(startkey=None, endkey={}, startkey_docid=None, from_date=None, u
 
     return payload
 
-def get_offset_payload(offset=None, keys=None):
+def get_offset_payload(offset=None, keys=None, maxResults=None):
     payload = {}
 
     if offset:
         payload["offset"] = offset
     if keys:
         payload["keys"] = keys
+    if maxResults:
+        payload["maxResults"] = maxResults
 
     return payload
 
 def get_token(serviceid, startkey=None, endkey={}, startkey_docid=None, from_date=None, until_date=None):
     return jwt.encode(get_payload(startkey, endkey, startkey_docid, from_date, until_date), serviceid, __JWT_ALG)
 
-def get_offset_token(serviceid, offset=None, keys=None):
-    return jwt.encode(get_offset_payload(offset, keys), serviceid, __JWT_ALG)
+def get_offset_token(serviceid, offset=None, keys=None, maxResults=None):
+    return jwt.encode(get_offset_payload(offset, keys, maxResults), serviceid, __JWT_ALG)
 
 
 if __name__ == "__main__":

LR/lr/model/base_model.py

@@ -33,6 +33,15 @@ def createDB(name, server=defaultCouchServer):
             #log.exception(ex)
         return server[name]
 
+    def getModelPasers(modelsPath):
+        modelParsers = {}
+
+        for path in modelsPath.split(','):
+            parser = ModelParser(path.strip())
+            modelParsers[parser.docVersion] = parser
+
+        return modelParsers
+
     class BaseModel(object):
         """Base model class for Learning Registry data models"""
 
@@ -41,8 +50,8 @@ class BaseModel(object):
         _SPEC_DATA = '_specData'
 
         _defaultDB = createDB(defaultDBName, server)
-        _modelParser = ModelParser(modelSpec)
-        
+        _modelParsers = getModelPasers(modelSpec)
+
         @classmethod
         def get(cls, doc_id, db=None):
             sourcDB = db
@@ -84,7 +93,7 @@ def __init__(self, data=None):
                         spec_data = json.loads(getFileString(data))
                     else:
                         spec_data = json.loads(data)
-                spec_data = {}                
+                spec_data = {}
                 spec_data.update(data)
                 # Remove _id an _rev in data if t there are present so that we can have
                 # a clean specData for validation
@@ -102,8 +111,8 @@ def __setattr__(self, name, value):
             # spec data attribute.
             if name == self._SPEC_DATA:
                 self.__dict__[self._SPEC_DATA] = value
-            elif name in self._modelParser.modelInfo.keys():
-                self._modelParser.validateField(name, value, self._specData)
+            elif self._isSpecDataFieldKey(name):
+                self._validateField(name, value)
                 self.__dict__[self._SPEC_DATA][name] = value
             elif name in self.__dict__.keys():
                 self.__dict__[name] = value
@@ -113,7 +122,7 @@ def __setattr__(self, name, value):
 
         def __getattr__(self, name):
             # Check if the attribute name is a spec attribute.
-            if name in self._modelParser.modelInfo.keys():
+            if self._isSpecDataFieldKey(name):
                 # If it is a spec attribute  and  it is set in the _specData  
                 # return it otherwise return None
                 return self.__dict__[self._SPEC_DATA].get(name)
@@ -123,18 +132,33 @@ def __getattr__(self, name):
                 raise AttributeError("'"+self.__class__.__name__+
                                                 "' object has no attribute'"+name+"'")
 
-
+        def _isSpecDataFieldKey(self, keyName):
+            #look through all the models spec for field name.
+            for specModel in self._modelParsers.values():
+                if keyName in specModel.modelInfo:
+                    return True
+            return False
+
+        def _validateField(self, fieldName, value):
+            # Look for a parser to validate the field against.  This done by using the
+            # version to look for the parser. If doc_version is not set already don't 
+            # don't anything.  The filed cannot be validate if we don't know what 
+            # doc_version to use
+            if self.doc_version in self._modelParsers:
+                self._modelParsers[self.doc_version].validateField(fieldName, value, self._specData) 
+
         def _preValidation(self):
             pass
 
         def _validate(self):
-            self._modelParser.validate(self._specData)
+           self._modelParsers[self.doc_version].validate(self._specData)
 
         def _postValidation(self):
             pass
 
         def toJSON(self):
             """Returns JSON object of the spec data"""
+
             return json.dumps(self._specData)
 
         def validate(self):