Changes associated with big archiver & qa merge. Gets archiver/qa inf…

…o from the package_dict rather than calling action functions to get the info.

ckanext/dgu/lib/helpers.py

@@ -15,7 +15,6 @@
 import datetime
 import random
 import types
-import json
 
 import ckan.plugins.toolkit as t
 c = t.c
@@ -40,11 +39,11 @@ def groupby(*args, **kwargs):
 def resource_as_json(resource):
     return json.dumps(resource)
 
-def is_resource_broken(resource_id):
-    from ckanext.archiver.model import Archival
-
-    archival = Archival.get_for_resource(resource_id)
-    return archival and archival.is_broken==True
+def is_resource_broken(resource):
+    archival = resource.get('archiver')
+    if not archival:
+        return None # don't know
+    return archival.get('is_broken')
 
 def _is_additional_resource(resource):
     """
@@ -66,20 +65,6 @@ def _is_individual_resource(resource):
     return not _is_additional_resource(resource) and \
            not _is_timeseries_resource(resource)
 
-# NB these 3 functions are overwritten by the other function of the same name,
-# but we should probably use these ones in preference
-def additional_resources(package):
-    """Extract the additional resources from a package"""
-    return filter(_is_additional_resource, package.get('resources'))
-
-def timeseries_resources(package):
-    """Extract the timeseries resources from a package"""
-    return filter(_is_timeseries_resource, package.get('resources'))
-
-def individual_resources(package):
-    """Extract the individual resources from a package"""
-    return filter(_is_individual_resource, package.get('resources'))
-
 def resource_type(resource):
     """
     Returns the resource type as a string.
@@ -432,15 +417,14 @@ def render_partial_datestamp(datestamp_str):
 
 
 def get_cache(resource_dict):
-    from ckanext.archiver.model import Archival
-    archival = Archival.get_for_resource(resource_dict['id'])
+    archival = resource_dict.get('archiver')
     if not archival:
         return (None, None)
-    url = (archival.cache_url or '').strip().replace('None', '')
+    url = (archival['cache_url'] or '').strip().replace('None', '')
     # strip off the domain, in case this is running in test
     # on a machine other than data.gov.uk
-    url = url.replace('http://data.gov.uk/', '/')
-    return url, archival.updated
+    url = re.sub('https?://data.gov.uk/', '/', url)
+    return url, render_datestamp(archival['updated'])
 
 
 # used in render_stars and read_common.html
@@ -461,33 +445,21 @@ def mini_stars_and_caption(num_stars):
     return t.literal('%s  %s' % (mini_stars, captions[num_stars]))
 
 # Used in read_common.html
-def calculate_dataset_stars(dataset_id):
-    from ckan.logic import get_action, NotFound
-    from ckan import model
+def calculate_dataset_stars(dataset_dict):
     if not is_plugin_enabled('qa'):
         return (0, '', '')
-    try:
-        context = {'model': model, 'session': model.Session}
-        qa = get_action('qa_package_openness_show')(context, {'id': dataset_id})
-    except NotFound:
-        return (0, '', '')
+    qa = dataset_dict.get('qa')
     if not qa:
         return (0, '', '')
     return (qa['openness_score'],
             qa['openness_score_reason'],
             qa['updated'])
 
 # Used in resource_read.html
-def render_resource_stars(resource_id):
-    from ckan.logic import get_action, NotFound
-    from ckan import model
+def render_resource_stars(resource_dict):
     if not is_plugin_enabled('qa'):
         return 'QA not installed'
-    try:
-        context = {'model': model, 'session': model.Session}
-        qa = get_action('qa_resource_show')(context, {'id': resource_id})
-    except NotFound:
-        return 'To be determined'
+    qa = resource_dict.get('qa')
     if not qa:
         return 'To be determined'
     return render_stars(qa['openness_score'], qa['openness_score_reason'],
@@ -504,24 +476,17 @@ def does_detected_format_disagree(detected_format, resource_format):
     return is_disagreement
 
 # used by resource_read.html
+# This is DGU's version of h.qa_openness_stars_resource_html
 def render_qa_info_for_resource(resource_dict):
-    resource_id = resource_dict['id']
-    from ckan.logic import get_action, NotFound
-    from ckan import model
     if not is_plugin_enabled('qa'):
         return 'QA not installed'
-    try:
-        context = {'model': model, 'session': model.Session}
-        qa = get_action('qa_resource_show')(context, {'id': resource_id})
-    except NotFound:
-        return 'To be determined'
+    qa = resource_dict.get('qa')
     if not qa:
         return 'To be determined'
     reason_list = (qa['openness_score_reason'] or '').replace('Reason: Download error. ', '').replace('Error details: ', '').split('. ')
-    resource = model.Resource.get(resource_id)
     ctx = {'qa': qa,
            'reason_list': reason_list,
-           'resource_format': resource.format,
+           'resource_format': resource_dict['format'],
            'resource_format_disagrees': does_detected_format_disagree(qa['format'], resource_dict['format']),
            'is_data': resource_dict['resource_type'] in ('file', None),
            }
@@ -1396,11 +1361,14 @@ def are_legacy_extras(data):
 
 def timeseries_resources():
     from ckan.lib.field_types import DateType
-    unsorted = c.pkg_dict.get('timeseries_resources', [])
+    unsorted = [res for res in c.pkg_dict.get('resources', [])
+                if _is_timeseries_resource(res)]
     get_iso_date = lambda resource: DateType.form_to_db(resource.get('date'),may_except=False)
     return sorted(unsorted, key=get_iso_date)
 
 def additional_resources():
+    return [res for res in c.pkg_dict.get('resources', [])
+            if _is_additional_resource(res)]
     return c.pkg_dict.get('additional_resources', [])
 
 def gemini_resources():
@@ -1423,7 +1391,8 @@ def gemini_resources():
     return gemini_resources
 
 def individual_resources():
-    r = c.pkg_dict.get('individual_resources', [])
+    r = [res for res in c.pkg_dict.get('resources', [])
+         if _is_individual_resource(res)]
     # In case the schema changes, the resources may or may not be split up into
     # three keys. So combine them if necessary
     if not r and not timeseries_resources() and not additional_resources():

ckanext/dgu/search_indexing.py

@@ -40,15 +40,15 @@ def add_inventory(cls, pkg_dict):
         ''' Sets unpublished to false if not present and also states whether the item is marked
             as never being published. '''
         pkg_dict['unpublished'] = pkg_dict.get('unpublished', False)
-        log.debug('Unpublished: %s', pkg_dict['unpublished'])
+        #log.debug('Unpublished: %s', pkg_dict['unpublished'])
 
         pkg_dict['core_dataset'] = pkg_dict.get('core-dataset', False)
-        log.debug('NII: %s', pkg_dict['core_dataset'])
+        #log.debug('NII: %s', pkg_dict['core_dataset'])
 
         # We also need to mark whether it is restricted (as in it will never be
         # released).
         pkg_dict['publish_restricted'] = pkg_dict.get('publish-restricted', False)
-        log.debug('Publish restricted: %s', pkg_dict['publish_restricted'])
+        #log.debug('Publish restricted: %s', pkg_dict['publish_restricted'])
 
 
     @classmethod
@@ -133,7 +133,7 @@ def add_field__group_abbreviation(cls, pkg_dict):
 
         if abbr:
             pkg_dict['group_abbreviation'] = abbr
-            log.debug('Abbreviations: %s', abbr)
+            #log.debug('Abbreviations: %s', abbr)
 
     @classmethod
     def add_field__publisher(cls, pkg_dict):
@@ -203,36 +203,24 @@ def add_field__harvest_document(cls, pkg_dict):
     @classmethod
     def add_field__openness(cls, pkg_dict):
         '''Add the openness score (stars) to the search index'''
-        import ckan
-        from ckanext.dgu.plugins_toolkit import get_action
-        context = {'model': ckan.model, 'session': ckan.model.Session,
-                   'ignore_auth': True}
-        data_dict = {'id': pkg_dict['id']}
-        try:
-            qa_openness = get_action('qa_package_openness_show')(context, data_dict)
-        except ObjectNotFound:
-            log.warning('No QA info for package %s', pkg_dict['name'])
+        archival = pkg_dict.get('archiver')
+        if not archival:
+            log.warning('No Archiver info for package %s', pkg_dict['name'])
             return
-        except KeyError:
-            # occurs during tests or if you've not install ckanext-qa
-            log.warning('QA not installed - not indexing it.')
+        qa = pkg_dict.get('qa')
+        if not qa:
+            log.warning('No QA info for package %s', pkg_dict['name'])
             return
-        pkg_dict['openness_score'] = qa_openness.get('openness_score')
+        pkg_dict['openness_score'] = qa.get('openness_score')
         log.debug('Openness score: %s', pkg_dict['openness_score'])
 
-        try:
-            qa_broken = get_action('qa_package_broken_show')(context, data_dict)
-        except ObjectNotFound:
-            log.warning('No brokenness info for package %s', pkg_dict['name'])
-            return
         if not hasattr(cls, 'broken_links_map'):
             cls.broken_links_map = {
                     True: 'Broken',
-                    'some': 'Partially broken',
                     False: 'OK',
                     None: 'TBC'
                     }
-        pkg_dict['broken_links'] = cls.broken_links_map[qa_broken.get('archival_is_broken')]
+        pkg_dict['broken_links'] = cls.broken_links_map[archival.get('is_broken')]
         log.debug('Broken links: %s', pkg_dict['broken_links'])
 
     @classmethod
@@ -268,7 +256,7 @@ def add_schema(cls, pkg_dict):
             except AttributeError, e:
                 log.error('Invalid schema_id: %r %s', schema_id, e)
         pkg_dict['schema_multi'] = schemas
-        log.debug('Schema: %s', ' '.join(schemas))
+        #log.debug('Schema: %s', ' '.join(schemas))
 
         try:
             codelist_ids = json.loads(pkg_dict.get('codelist') or '[]')
@@ -280,4 +268,4 @@ def add_schema(cls, pkg_dict):
         for codelist_id in codelist_ids:
             codelists.append(Codelist.get(codelist_id).title)
         pkg_dict['codelist_multi'] = codelists
-        log.debug('Code lists: %s', ' '.join(codelists))
+        #log.debug('Code lists: %s', ' '.join(codelists))

ckanext/dgu/theme/templates/package/read.html

@@ -318,7 +318,7 @@ <h2>Issues</h2>
       <div class="dataset-resource-text">
         <div class="inner">
           <div class="inner2">
-            {% if h.is_resource_broken(res['id']) %}
+            {% if h.is_resource_broken(res) %}
               <a class="js-tooltip" data-toggle="tooltip" data-placement="right" title="There is an error with this resource" href="{{h.url_for(controller='package', action='resource_read', id=c.pkg_dict['name'], resource_id=res['id'])}}#qualitycheck" style="text-decoration:none;color:#D43F3A;">
                 <i class="icon-exclamation-sign"></i>
               </a>
@@ -370,21 +370,21 @@ <h2>Issues</h2>
             {% set cache_url, cache_timestamp = h.get_cache(res) %}
             {% if not gemini and cache_url %}
             <li>
-              <a href="{{cache_url}}" class="js-tooltip" data-placement="bottom" title="Cached&nbsp;by&nbsp;data.gov.uk&nbsp;on {{cache_timestamp.strftime('%d/%m/%Y') if cache_timestamp else 'unknown date'}}" onclick="{{h.ga_download_tracking(res, publisher_name, 'download-cache')}}">
+              <a href="{{cache_url}}" class="js-tooltip" data-placement="bottom" title="Cached&nbsp;by&nbsp;data.gov.uk&nbsp;on {{cache_timestamp if cache_timestamp else 'unknown date'}}" onclick="{{h.ga_download_tracking(res, publisher_name, 'download-cache')}}">
                 <i class="icon-save"></i>&nbsp;
                 Cached
               </a>
             </li>
             {% endif %}
+            {% if not gemini and not cache_url %}
+              <li>
+                <a class="disabled">
+                  <i class="icon-save"></i>&nbsp;
+                  (no cache available)
+                </a>
+              </li>
+            {% endif %}
           {% endwith %}
-          {% if not gemini and not h.get_cache(res)[0] %}
-            <li>
-              <a class="disabled">
-                <i class="icon-save"></i>&nbsp;
-                (no cache available)
-              </a>
-            </li>
-          {% endif %}
         </ul><!-- /dropdown-mnu-->
         {% endwith %}
       </div><!--/dropdown-->

ckanext/dgu/theme/templates/package/read_common.html

@@ -128,7 +128,7 @@ <h1>{{c.pkg_dict.get('title', c.pkg_dict['name'])}} {%if h.is_unpublished_item(c
     </div>
 
     <div class="dataset-stars h1-subheading" style="margin-top: 0px;margin-bottom: 12px;">
-      {% with stars,reason,last_updated = h.calculate_dataset_stars(c.pkg.id) %}
+      {% with stars,reason,last_updated = h.calculate_dataset_stars(c.pkg_dict) %}
       Openness rating:
       <span class="star-rating">
         <span class="tooltip">

ckanext/dgu/theme/templates/package/read_common_jinja.html

@@ -132,7 +132,7 @@ <h1>{{c.pkg_dict.get('title', c.pkg_dict['name'])}} {%if h.is_unpublished_item(c
     </div>
 
     <div class="dataset-stars h1-subheading" style="margin-top: 0px;margin-bottom: 12px;">
-      {% with stars,reason,last_updated = h.calculate_dataset_stars(c.pkg.id) %}
+      {% with stars,reason,last_updated = h.calculate_dataset_stars(c.pkg_dict) %}
       Openness rating:
       <span class="star-rating">
         <span class="tooltip">

ckanext/dgu/theme/templates/package/resource_read.html

@@ -82,7 +82,7 @@
       {% set cache_url, cache_timestamp = h.get_cache(c.resource) %}
       {% if cache_url and not h.get_resource_wms(c.resource) %}
        <a class="btn btn-danger resource-url-analytics resource-type-{{c.resource.get('resource_type')}}" href="{{cache_url}}" onclick="{{h.ga_download_tracking(c.resource, publisher.name, 'download-cache')}}">
-        <div class="download js-tooltip" data-placement="bottom" data-original-title="Cached by data.gov.uk on: {{cache_timestamp.strftime('%d/%m/%Y') if cache_timestamp else 'unknown'}}">
+        <div class="download js-tooltip" data-placement="bottom" data-original-title="Cached by data.gov.uk on: {{cache_timestamp if cache_timestamp else 'unknown'}}">
           <i class="icon-save"></i>&nbsp; Cached
         </div>
        </a>
@@ -134,11 +134,12 @@ <h2>Preview</h2>
       <tr>
         <!--! This is data, so show 5 stars of openness -->
         <td class="key">Resource Openness:</td>
-        <td class="value">{{h.render_resource_stars(c.resource['id'])}}</td>
+        <td class="value">{{h.render_resource_stars(c.resource)}}</td>
       </tr>
       {% endif %}
       <tr>
         <td class="key">Quality Check:<a name="qualitycheck"></a></td>
+        {# render_qa_info_for_resource is DGU's version of h.qa_openness_stars_resource_html #}
         <td class="value">{{h.render_qa_info_for_resource(c.resource)}}</td>
       </tr>
       {% for field_dict, label_attributes, value_attributes in h.get_resource_fields(c.resource, h.as_dict(c.pkg_extras)) %}