Merge pull request #1 from zotya/master

Data persistence for sparql results & quick preview for queries

docs/HISTORY.txt

@@ -3,6 +3,14 @@ Changelog
 
 2.4-dev - (unreleased)
 ----------------------
+* Upgrade step: Within "Plone > Site setup > Add-ons" click on
+  upgrade button available for eea.sparql.
+* Feature: Added quick preview for sparql queries
+  [szabozo0 refs #9608]
+* Feature: Data persistence and versioning
+    - Added versioning for sparql results
+    - Possibility to select static & live queries
+  [szabozo0 refs #9424]
 
 2.3 - (2013-01-18)
 ------------------

eea/sparql/browser/configure.zcml

@@ -151,4 +151,20 @@
         permission="cmf.ModifyPortalContent"
         />
 
+    <browser:page
+        name="sparql.quick_preview"
+        for="*"
+        class=".sparql.QuickPreview"
+        attribute="preview"
+        permission="zope2.View"
+        />
+
+    <browser:page
+        name="sparql.related_items"
+        for="*"
+        class=".sparql.Sparql"
+        attribute="relatedItems"
+        permission="zope2.View"
+        />
+
 </configure>

eea/sparql/browser/sparql.py

@@ -6,6 +6,10 @@
 from Products.ZSPARQLMethod.Method import interpolate_query_html
 from Products.ZSPARQLMethod.Method import map_arg_values
 from Products.ZSPARQLMethod.Method import parse_arg_spec
+from Products.ZSPARQLMethod.Method import interpolate_query
+from Products.ZSPARQLMethod.Method import run_with_timeout
+from Products.ZSPARQLMethod.Method import query_and_get_result
+
 from eea.sparql.converter.sparql2json import sparql2json
 from eea.sparql.converter.sparql2json import sortProperties
 from eea.versions import versions
@@ -14,6 +18,7 @@
 import json
 import urllib2
 import contextlib
+import cgi
 
 logger = logging.getLogger('eea.sparql')
 
@@ -163,11 +168,10 @@ def sparql2csv(self, dialect='excel'):
             header = '%s:%s' % (col, data['properties'][col]['valueType'])
             row.append(header)
         writter.writerow(row)
-
         for item in data['items']:
             row = []
             for col in headers:
-                row.append(unicode(item[col]))
+                row.append(unicode(item[col]).encode('utf'))
             writter.writerow(row)
 
         return ''
@@ -240,6 +244,11 @@ def isDavizInstalled(self):
 
         return has_daviz
 
+    def relatedItems(self):
+        """ Items what are back related to this query
+        """
+        return json.dumps([[x.title, x.absolute_url()]
+                            for x in self.context.getBRefs()])
 
 class SparqlBookmarksFolder(Sparql):
     """SparqlBookmarksFolder view"""
@@ -312,3 +321,61 @@ def __call__(self):
             except Exception, err:
                 logger.exception(err)
         return "Sync done"
+
+class QuickPreview(BrowserView):
+    """ Quick Preview For Query
+    """
+
+    def preview(self):
+        """preview"""
+        tmp_query = self.request.get("sparql_query", "")
+        tmp_arg_spec = self.request.get("arg_spec", "")
+        tmp_endpoint = self.request.get("endpoint", "")
+        tmp_timeout = int(self.request.get("timeout", "0"))
+
+        arg_spec = parse_arg_spec(tmp_arg_spec)
+        missing, arg_values = map_arg_values(arg_spec, self.request.form)
+        error = None
+        if missing:
+            error = ""
+            for missing_arg in missing:
+                error = error + "<div>Argument '%s' missing</div>" % missing_arg
+        else:
+            result = {}
+            data = []
+            error = None
+            cooked_query = interpolate_query(tmp_query, arg_values)
+            args = (tmp_endpoint, cooked_query)
+            try:
+                result, error = {}, None
+                result = run_with_timeout(tmp_timeout,
+                                            query_and_get_result,
+                                            *args)
+                data = result.get('result')
+                error = error or result.get('exception')
+            except Exception:
+                import traceback
+                error = traceback.format_exc()
+
+        if error:
+            return "<blockquote class='sparql-error'> %s </blockquote>" % error
+
+        result = []
+        result.append(u"<table class='sparql-results'>")
+        result.append(u"<thead>")
+        result.append(u"<tr>")
+        for var_name in data.get('var_names', []):
+            result.append(u"<th> %s </th>" %var_name)
+        result.append(u"</tr>")
+        result.append(u"</thead>")
+        result.append(u"<tbody>")
+        for row in data.get('rows', []):
+            result.append(u"<tr class='row_0'>")
+            for value in row:
+                result.append(u"<td> %s </td>" %cgi.escape(value.n3()))
+            result.append(u"</tr>")
+        result.append(u"</tbody>")
+        result.append(u"</table>")
+        return "\n".join(result)
+
+

eea/sparql/browser/view.pt

@@ -36,7 +36,8 @@
     </div>
 
     <h3 i18n:translate="">Dataset preview</h3>
-
+    <h4 tal:condition="context/sparql_static">(Static Query)</h4>
+    <h4 tal:condition="not:context/sparql_static">(Live Query)</h4>
     <div tal:define="data options/data">
       <div class="eea-sparql-table" style="overflow: hidden">
         <metal:table use-macro="here/@@sparql.preview/macros/table" />

eea/sparql/cache/browser.py

@@ -24,11 +24,14 @@ class CacheView(BrowserView):
     """ Caching for sparql query results
     """
     def __call__(self):
-        if not "submit" in self.request.form:
-            return self.index()
+        if "invalidate_cache" in self.request.form:
+            event.notify(ObjectModifiedEvent(self.context))
+            IStatusMessage(self.request).addStatusMessage("Cache invalidated")
+        if "invalidate_last_working_results" in self.request.form:
+            self.context.invalidateWorkingResult()
+            message = "Last working results invalidated"
+            IStatusMessage(self.request).addStatusMessage(message)
 
-        event.notify(ObjectModifiedEvent(self.context))
-        IStatusMessage(self.request).addStatusMessage("Cache invalidated")
         return self.index()
 
 def purgeRelatedItems(obj, evt):

eea/sparql/cache/caching.pt

@@ -12,7 +12,10 @@
     <div metal:fill-slot="main" >
       <form method="POST" action="" tal:attributes="action string:${context/absolute_url}/@@caching">
 
-        <input type="submit" name="submit" value="Invalidate cache"
+        <input type="submit" name="invalidate_cache" value="Invalidate cache"
+               i18n:attributes="value"  />
+        <br/>
+        <input type="submit" name="invalidate_last_working_results" value="Invalidate last working result"
                i18n:attributes="value"  />
 
       </form>

eea/sparql/configure.zcml

@@ -6,6 +6,7 @@
 
   <include file="profiles.zcml" />
   <include file="permissions.zcml" />
+  <include file="skins.zcml" />
 
   <include package=".browser" />
   <include package=".cache" />

eea/sparql/content/sparql.py

@@ -9,13 +9,14 @@
 from Products.Archetypes.atapi import IntegerField
 from Products.Archetypes.atapi import SelectionWidget
 from Products.Archetypes.atapi import Schema
-from Products.Archetypes.atapi import StringField, StringWidget
+from Products.Archetypes.atapi import StringField, StringWidget, \
+                                        BooleanWidget, BooleanField
 from Products.Archetypes.atapi import TextField, TextAreaWidget
 from Products.ZSPARQLMethod.Method import ZSPARQLMethod, \
                                         interpolate_query, \
                                         run_with_timeout, \
-                                        query_and_get_result, \
                                         parse_arg_spec, \
+                                        query_and_get_result, \
                                         map_arg_values
 from AccessControl.Permissions import view
 from eea.sparql.cache import ramcache, cacheSparqlKey
@@ -26,6 +27,14 @@
 
 from zope.interface import implements
 
+from Products.CMFCore.utils import getToolByName
+from Products.CMFEditions.interfaces.IModifier import FileTooLargeToVersionError
+
+from AccessControl import SpecialUsers
+from AccessControl import getSecurityManager
+from AccessControl.SecurityManagement import newSecurityManager, \
+                                            setSecurityManager
+
 SparqlBaseSchema = atapi.Schema((
     StringField(
         name='endpoint_url',
@@ -59,10 +68,31 @@
         allowable_content_types = ('text/plain',),
 
         widget=TextAreaWidget(
+            macro="sparql_textfield_with_preview",
+            helper_js=("sparql_textfield_with_preview.js",),
+            helper_css=("sparql_textfield_with_preview.css",),
             label="Query",
         ),
         required=1
     ),
+    BooleanField(
+        name='sparql_static',
+        widget=BooleanWidget(
+            label='Static query',
+            description='The data will be fetched only once',
+        ),
+        default=False,
+        required=0
+    ),
+    TextField(
+        name='sparql_results',
+        widget=TextAreaWidget(
+            label="Results",
+            visible={'edit': 'invisible', 'view': 'invisible' }
+        ),
+        required=0,
+
+    ),
 ))
 
 SparqlSchema = getattr(base.ATCTContent, 'schema', Schema(())).copy() + \
@@ -77,6 +107,8 @@
         SparqlBaseSchema.copy()
 SparqlBookmarksFolderSchema['sparql_query'].widget.description = \
         'The query should return label, bookmark url, query'
+SparqlBookmarksFolderSchema['sparql_static'].widget.visible['edit'] = \
+        'invisible'
 
 class Sparql(base.ATCTContent, ZSPARQLMethod):
     """Sparql"""
@@ -127,25 +159,86 @@ def setTimeout(self, value):
         except Exception:
             self.timeout = 10
 
+    security.declareProtected(view, 'invalidateWorkingResult')
+    def invalidateWorkingResult(self):
+        """ invalidate working results"""
+        self.cached_result = {}
+        self.setSparql_results("")
+        pr = getToolByName(self, 'portal_repository')
+        comment = "Invalidated last working result"
+        comment = comment.encode('utf')
+        try:
+            pr.save(obj=self, comment=comment)
+        except FileTooLargeToVersionError:
+            commands = view.getCommandSet('plone')
+            commands.issuePortalMessage(
+                """Changes Saved. Versioning for this file 
+                   has been disabled because it is too large.""",
+                msgtype="warn")
+
     security.declareProtected(view, 'execute')
     def execute(self, **arg_values):
         """
         Override execute from ZSPARQLMethod in order to have a default timeout
         """
+        cached_result = getattr(self, 'cached_result', {})
         cooked_query = interpolate_query(self.query, arg_values)
-        cache_key = {'query': cooked_query}
-        result = self.ZCacheable_get(keywords=cache_key)
 
-        if result is None:
+        force_requery = False
+
+        if not self.getSparql_static():
+            force_requery = True
+
+        if cached_result == {}:
+            force_requery = True
+
+        if force_requery:
             args = (self.endpoint_url, cooked_query)
-            result = run_with_timeout(
+            new_result = run_with_timeout(
                 max(getattr(self, 'timeout', 10), 10),
                 query_and_get_result,
                 *args)
-            self.ZCacheable_set(result, keywords=cache_key)
-
-        return result
 
+            force_save = False
+
+            if new_result.get("result", {}) != {}:
+                if new_result != cached_result:
+                    if len(new_result.get("result", {}).get("rows", {})) > 0:
+                        force_save = True
+                    else:
+                        if len(cached_result.get('result', {}).\
+                            get('rows', {})) == 0:
+                            force_save = True
+
+            if force_save:
+                self.cached_result = new_result
+                new_sparql_results = u""
+                for row in self.cached_result.get('result', {}).get('rows', {}):
+                    for val in row:
+                        new_sparql_results = new_sparql_results + \
+                            unicode(val) + " | "
+                    new_sparql_results = new_sparql_results[0:-3] + "\n"
+                    self.setSparql_results(new_sparql_results)
+                pr = getToolByName(self, 'portal_repository')
+                if self.portal_type in pr.getVersionableContentTypes():
+                    comment = "Result changed"
+                    comment = comment.encode('utf')
+
+                    oldSecurityManager = getSecurityManager()
+                    newSecurityManager(self.REQUEST, SpecialUsers.system)
+                    try:
+                        pr.save(obj=self, comment=comment)
+                    except FileTooLargeToVersionError:
+                        commands = view.getCommandSet('plone')
+                        commands.issuePortalMessage(
+                            """Changes Saved. Versioning for this file 
+                               has been disabled because it is too large.""",
+                            msgtype="warn")
+                    setSecurityManager(oldSecurityManager)
+
+            if new_result.get('exception', None):
+                self.cached_result['exception'] = new_result['exception']
+        return self.cached_result
 
 class SparqlBookmarksFolder(ATFolder, Sparql):
     """Sparql Bookmarks Folder"""

eea/sparql/profiles/default/diff_tool.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<object>
+ <difftypes>
+  <type portal_type="Sparql">
+   <field name="any" difftype="Compound Diff for AT types"/>
+  </type>
+ </difftypes>
+</object>

eea/sparql/profiles/default/metadata.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 <metadata>
-  <version>1001</version>
+  <version>2400</version>
   <dependencies>
     <dependency>profile-eea.versions:default</dependency>
   </dependencies>

eea/sparql/profiles/default/repositorytool.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<repositorytool>
+ <policymap>
+  <type name="Sparql">
+   <policy name="at_edit_autoversion"/>
+   <policy name="version_on_revert"/>
+  </type>
+ </policymap>
+</repositorytool>

eea/sparql/profiles/default/skins.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0"?>
+<object name="portal_skins">
+  <object name="sparql_templates"
+    meta_type="Filesystem Directory View"
+    directory="eea.sparql:skins/sparql_templates"/>
+
+  <skin-path name="*">
+    <layer insert-after="custom" name="sparql_templates"/>
+  </skin-path>
+</object>