Merged master

ckan/config/deployment.ini_tmpl

@@ -44,6 +44,11 @@ sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckantest
 #sqlalchemy.url = sqlite:///
 #sqlalchemy.url = sqlite:///%(here)s/somedb.db
 
+## Datastore
+## Uncommment to set the datastore urls
+#ckan.datastore.write_url = postgresql://ckanuser:pass@localhost/ckantest
+#ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/ckantest
+
 # repoze.who config
 who.config_file = %(here)s/who.ini
 who.log_level = warning
@@ -226,10 +231,6 @@ ckan.feeds.author_link =
 #ofs.aws_access_key_id = ....
 #ofs.aws_secret_access_key = ....
 
-## Webstore
-## Uncommment to enable datastore
-# ckan.datastore.enabled = 1
-
 ## ===================================
 ## Extensions
 

ckan/config/environment.py

@@ -126,7 +126,7 @@ def find_controller(self, controller):
     paths = dict(root=root,
                  controllers=os.path.join(root, 'controllers'),
                  static_files=os.path.join(root, 'public'),
-                 templates=[os.path.join(root, 'templates')])
+                 templates=[])
 
     # Initialize config with the basic options
 
@@ -189,15 +189,13 @@ def find_controller(self, controller):
     ## redo template setup to use genshi.search_path
     ## (so remove std template setup)
     legacy_templates_path = os.path.join(root, 'templates_legacy')
+    jinja2_templates_path = os.path.join(root, 'templates')
     if asbool(config.get('ckan.legacy_templates', 'no')):
-        template_paths = [legacy_templates_path]
-        # if we are testing allow new templates
-        if asbool(config.get('ckan.enable_testing', 'false')):
-            jinja2_templates_path = os.path.join(root, 'templates')
-            template_paths.append(jinja2_templates_path)
+        # We want the new template path for extra snippets like the
+        # dataviewer and also for some testing stuff
+        template_paths = [legacy_templates_path, jinja2_templates_path]
     else:
-        template_paths = [paths['templates'][0]]
-        template_paths.append(legacy_templates_path)
+        template_paths = [jinja2_templates_path, legacy_templates_path]
 
     extra_template_paths = config.get('extra_template_paths', '')
     if extra_template_paths:

ckan/config/routing.py

@@ -213,6 +213,8 @@ def make_map():
                   action='resource_embedded_dataviewer')
         m.connect('/dataset/{id}/resource/{resource_id}/viewer',
                   action='resource_embedded_dataviewer', width="960", height="800")
+        m.connect('/dataset/{id}/resource/{resource_id}/preview/{preview_type}',
+                  action='resource_datapreview')
 
     # group
     map.redirect('/groups', '/group')

ckan/controllers/package.py

@@ -310,7 +310,6 @@ def read(self, id, format='html'):
         try:
             c.pkg_dict = get_action('package_show')(context, data_dict)
             c.pkg = context['package']
-            c.resources_json = json.dumps(c.pkg_dict.get('resources', []))
         except NotFound:
             abort(404, _('Dataset not found'))
         except NotAuthorized:
@@ -1146,7 +1145,6 @@ def resource_read(self, id, resource_id):
             c.package = get_action('package_show')(context, {'id': id})
             # required for nav menu
             c.pkg = context['package']
-            c.resource_json = json.dumps(c.resource)
             c.pkg_dict = c.package
         except NotFound:
             abort(404, _('Resource not found'))
@@ -1159,8 +1157,9 @@ def resource_read(self, id, resource_id):
                 get_license_register()[license_id].isopen()
         except KeyError:
             c.package['isopen'] = False
-        c.datastore_api = h.url_for('datastore_read', id=c.resource.get('id'),
-                                    qualified=True)
+
+        # TODO: find a nicer way of doing this
+        c.datastore_api = '%s/api/action' % config.get('ckan.site_url','').rstrip('/')
 
         c.related_count = c.pkg.related_count
         return render('package/resource_read.html')
@@ -1280,3 +1279,21 @@ def _parse_recline_state(self, params):
                     not k.endswith(recline_state['currentView']):
                 recline_state.pop(k)
         return recline_state
+
+    def resource_datapreview(self, id, resource_id, preview_type):
+        '''
+        Embeded page for a resource data-preview.
+        '''
+        context = {'model': model, 'session': model.Session,
+                   'user': c.user or c.author}
+
+        try:
+            c.resource = get_action('resource_show')(context,
+                                                     {'id': resource_id})
+            c.package = get_action('package_show')(context, {'id': id})
+            c.resource_json = json.dumps(c.resource)
+        except NotFound:
+            abort(404, _('Resource not found'))
+        except NotAuthorized:
+            abort(401, _('Unauthorized to read resource %s') % id)
+        return render('dataviewer/{type}.html'.format(type=preview_type))

ckan/lib/cli.py

@@ -13,6 +13,35 @@
 #   i.e. do the imports in methods, after _load_config is called.
 #   Otherwise loggers get disabled.
 
+
+def parse_db_config(config_key='sqlalchemy.url'):
+    ''' Takes a config key for a database connection url and parses it into
+    a dictionary. Expects a url like:
+
+    'postgres://tester:pass@localhost/ckantest3'
+    '''
+    from pylons import config
+    url = config[config_key]
+    regex = [
+        '^\s*(?P<db_type>\w*)',
+        '://',
+        '(?P<db_user>[^:]*)',
+        ':?',
+        '(?P<db_pass>[^@]*)',
+        '@',
+        '(?P<db_host>[^/:]*)',
+        ':?',
+        '(?P<db_port>[^/]*)',
+        '/',
+        '(?P<db_name>[\w.-]*)'
+    ]
+    db_details_match = re.match(''.join(regex), url)
+    if not db_details_match:
+        raise Exception('Could not extract db details from url: %r' % url)
+    db_details = db_details_match.groupdict()
+    return db_details
+
+
 class MockTranslator(object):
     def gettext(self, value):
         return value
@@ -134,14 +163,7 @@ def command(self):
             sys.exit(1)
 
     def _get_db_config(self):
-        from pylons import config
-        url = config['sqlalchemy.url']
-        # e.g. 'postgres://tester:pass@localhost/ckantest3'
-        db_details_match = re.match('^\s*(?P<db_type>\w*)://(?P<db_user>[^:]*):?(?P<db_pass>[^@]*)@(?P<db_host>[^/:]*):?(?P<db_port>[^/]*)/(?P<db_name>[\w.-]*)', url)
-        if not db_details_match:
-            raise Exception('Could not extract db details from url: %r' % url)
-        db_details = db_details_match.groupdict()
-        return db_details
+        return parse_db_config()
 
     def _get_postgres_cmd(self, command):
         self.db_details = self._get_db_config()

ckan/lib/fanstatic_resources.py

@@ -248,10 +248,10 @@ def create_resource(path, lib_name, count, inline=False):
 
 create_library('vendor', os.path.join(base_path, 'vendor'), depend_base=False)
 
-create_library('datapreview', os.path.join(base_path, 'datapreview'),
+create_library('base', os.path.join(base_path, 'javascript'),
                depend_base=False)
 
-create_library('base', os.path.join(base_path, 'javascript'),
+create_library('datapreview', os.path.join(base_path, 'datapreview'),
                depend_base=False)
 
 create_library('css', os.path.join(base_path, 'css'), depend_base=False)

ckan/lib/helpers.py

@@ -1163,6 +1163,35 @@ def include_resource(resource):
     r = getattr(fanstatic_resources, resource)
     r.need()
 
+def urls_for_resource(resource):
+    ''' Returns a list of urls for the resource specified.  If the resource
+    is a group or has dependencies then there can be multiple urls.
+
+    NOTE: This is for special situations only and is not the way to generaly
+    include resources.  It is advised not to use this function.'''
+    r = getattr(fanstatic_resources, resource)
+    resources = list(r.resources)
+    core = fanstatic_resources.fanstatic_extensions.core
+    f = core.get_needed()
+    lib = resources[0].library
+    root_path = f.library_url(lib)
+
+    resources = core.sort_resources(resources)
+    if f._bundle:
+        resources = core.bundle_resources(resources)
+    out = []
+    for resource in resources:
+        if isinstance(resource, core.Bundle):
+            paths = [resource.relpath for resource in resource.resources()]
+            relpath = ';'.join(paths)
+            relpath = core.BUNDLE_PREFIX + relpath
+        else:
+            relpath = resource.relpath
+
+        out.append('%s/%s' % (root_path, relpath))
+    return out
+
+
 def debug_inspect(arg):
     ''' Output pprint.pformat view of supplied arg '''
     return literal('<pre>') + pprint.pformat(arg) + literal('</pre>')
@@ -1309,6 +1338,53 @@ def format_resource_items(items):
     return sorted(output, key=lambda x:x[0])
 
 
+def resource_preview(resource, pkg_id):
+    '''
+    Returns a rendered snippet for a embeded resource preview.
+
+    Depending on the type, different previews are loaded.
+    This could be an img tag where the image is loaded directly or an iframe that
+    embeds a webpage, recline or a pdf preview.
+    '''
+
+    DIRECT_EMBEDS = ['png', 'jpg', 'gif']
+    LOADABLE = ['html', 'htm', 'rdf+xml', 'owl+xml', 'xml', 'n3',
+                'n-triples', 'turtle', 'plain', 'atom', 'tsv', 'rss',
+                'txt', 'json']
+    PDF = ['pdf', 'x-pdf', 'acrobat', 'vnd.pdf']
+
+    format_lower = resource['format'].lower()
+    directly = False
+    url = ''
+
+    if resource.get('datastore_active') or format_lower in ['csv', 'xls', 'tsv']:
+        url = url_for(controller='package', action='resource_datapreview',
+            resource_id=resource['id'], preview_type='recline', id=pkg_id, qualified=True)
+    elif format_lower in PDF:
+        url = url_for(controller='package', action='resource_datapreview',
+            resource_id=resource['id'], preview_type='pdf', id=pkg_id, qualified=True)
+    elif format_lower == 'jsonp':
+        url = url_for(controller='package', action='resource_datapreview',
+            resource_id=resource['id'], preview_type='json', id=pkg_id, qualified=True)
+    elif format_lower in LOADABLE:
+        url = resource['url']
+    elif format_lower in DIRECT_EMBEDS:
+        directly = True
+        url = resource['url']
+    else:
+        log.info('no handler for {}'.format(resource['format']))
+        return snippet(
+            "dataviewer/snippets/no_preview.html",
+            resource_type=format_lower
+            )
+
+    return snippet(
+        "dataviewer/snippets/data_preview.html",
+        embed=directly,
+        resource_url=url
+        )
+
+
 # these are the functions that will end up in `h` template helpers
 # if config option restrict_template_vars is true
 __allowed_functions__ = [
@@ -1363,6 +1439,7 @@ def format_resource_items(items):
            'get_facet_items_dict',
            'unselected_facet_items',
            'include_resource',
+           'urls_for_resource',
            'build_nav_main',
            'debug_inspect',
            'dict_list_reduce',
@@ -1383,6 +1460,7 @@ def format_resource_items(items):
            'get_request_param',
            'render_markdown',
            'format_resource_items',
+           'resource_preview',
            # imported into ckan.lib.helpers
            'literal',
            'link_to',

ckan/public/base/datapreview/css/json.css

@@ -0,0 +1,33 @@
+body {
+	width: 500px;
+}
+
+pre {
+	font-size: 13px;
+}
+
+.loading {
+	font-weight: bold;
+	font-family: sans-serif;
+	font-size: 16px;
+	position: fixed;
+	left: -20px;
+	top: 20px;
+}
+
+.string {
+	color: #009900;
+}
+.number {
+	color: #0066FF;
+}
+.boolean {
+	color: #E62E00;
+}
+.null {
+	color: #E62E00;
+}
+.key {
+	color: #222;
+	font-weight: bold;
+}

ckan/public/base/datapreview/css/pdf.css

@@ -0,0 +1,4 @@
+body {
+  height: 500px;
+  overflow: hidden;
+}

ckan/public/base/datapreview/css/datapreview.css → ckan/public/base/datapreview/css/recline.css

@@ -1,3 +1,12 @@
+body {
+  background-color: #fff;
+}
+
+.recline-data-explorer{
+  position: relative;
+  overflow: auto;
+}
+
 .recline-query-editor form, .recline-query-editor .text-query {
   height: 28px;
 }
@@ -36,7 +45,6 @@
   border-radius: 0 3px 3px 0;
 }
 
-
 .recline-slickgrid {
   height: 550px;
 }
@@ -51,8 +59,13 @@
   vertical-align: top;
 }
 
-#ckanext-datapreview iframe {
-  min-height: 480px;
-  border: 1px solid #ccc;
-  padding: 7px;
+.loading-dialog {
+  width: 120px;
+}
+
+.loading-spinner {
+  float: right;
+  background-image: url('../img/ajaxload-circle.gif');
+  width: 32px;
+  height: 32px;
 }