Merge branch 'master' into 2939-orgs

ckan/config/deployment.ini_tmpl

@@ -44,6 +44,11 @@ sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckantest
 #sqlalchemy.url = sqlite:///
 #sqlalchemy.url = sqlite:///%(here)s/somedb.db
 
+## Datastore
+## Uncommment to set the datastore urls
+#ckan.datastore.write_url = postgresql://ckanuser:pass@localhost/ckantest
+#ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/ckantest
+
 # repoze.who config
 who.config_file = %(here)s/who.ini
 who.log_level = warning
@@ -226,10 +231,6 @@ ckan.feeds.author_link =
 #ofs.aws_access_key_id = ....
 #ofs.aws_secret_access_key = ....
 
-## Webstore
-## Uncommment to enable datastore
-# ckan.datastore.enabled = 1
-
 ## ===================================
 ## Extensions
 

ckan/config/environment.py

@@ -126,7 +126,7 @@ def find_controller(self, controller):
     paths = dict(root=root,
                  controllers=os.path.join(root, 'controllers'),
                  static_files=os.path.join(root, 'public'),
-                 templates=[os.path.join(root, 'templates')])
+                 templates=[])
 
     # Initialize config with the basic options
 
@@ -189,15 +189,13 @@ def find_controller(self, controller):
     ## redo template setup to use genshi.search_path
     ## (so remove std template setup)
     legacy_templates_path = os.path.join(root, 'templates_legacy')
+    jinja2_templates_path = os.path.join(root, 'templates')
     if asbool(config.get('ckan.legacy_templates', 'no')):
-        template_paths = [legacy_templates_path]
-        # if we are testing allow new templates
-        if asbool(config.get('ckan.enable_testing', 'false')):
-            jinja2_templates_path = os.path.join(root, 'templates')
-            template_paths.append(jinja2_templates_path)
+        # We want the new template path for extra snippets like the
+        # dataviewer and also for some testing stuff
+        template_paths = [legacy_templates_path, jinja2_templates_path]
     else:
-        template_paths = [paths['templates'][0]]
-        template_paths.append(legacy_templates_path)
+        template_paths = [jinja2_templates_path, legacy_templates_path]
 
     extra_template_paths = config.get('extra_template_paths', '')
     if extra_template_paths:

ckan/config/routing.py

@@ -209,6 +209,8 @@ def make_map():
                   action='resource_embedded_dataviewer')
         m.connect('/dataset/{id}/resource/{resource_id}/viewer',
                   action='resource_embedded_dataviewer', width="960", height="800")
+        m.connect('/dataset/{id}/resource/{resource_id}/preview/{preview_type}',
+                  action='resource_datapreview')
 
     # group
     map.redirect('/groups', '/group')

ckan/controllers/package.py

@@ -308,7 +308,6 @@ def read(self, id, format='html'):
         try:
             c.pkg_dict = get_action('package_show')(context, data_dict)
             c.pkg = context['package']
-            c.resources_json = json.dumps(c.pkg_dict.get('resources', []))
         except NotFound:
             abort(404, _('Dataset not found'))
         except NotAuthorized:
@@ -1144,7 +1143,6 @@ def resource_read(self, id, resource_id):
             c.package = get_action('package_show')(context, {'id': id})
             # required for nav menu
             c.pkg = context['package']
-            c.resource_json = json.dumps(c.resource)
             c.pkg_dict = c.package
         except NotFound:
             abort(404, _('Resource not found'))
@@ -1157,8 +1155,9 @@ def resource_read(self, id, resource_id):
                 get_license_register()[license_id].isopen()
         except KeyError:
             c.package['isopen'] = False
-        c.datastore_api = h.url_for('datastore_read', id=c.resource.get('id'),
-                                    qualified=True)
+
+        # TODO: find a nicer way of doing this
+        c.datastore_api = '%s/api/action' % config.get('ckan.site_url','').rstrip('/')
 
         c.related_count = c.pkg.related_count
         return render('package/resource_read.html')
@@ -1278,3 +1277,21 @@ def _parse_recline_state(self, params):
                     not k.endswith(recline_state['currentView']):
                 recline_state.pop(k)
         return recline_state
+
+    def resource_datapreview(self, id, resource_id, preview_type):
+        '''
+        Embeded page for a resource data-preview.
+        '''
+        context = {'model': model, 'session': model.Session,
+                   'user': c.user or c.author}
+
+        try:
+            c.resource = get_action('resource_show')(context,
+                                                     {'id': resource_id})
+            c.package = get_action('package_show')(context, {'id': id})
+            c.resource_json = json.dumps(c.resource)
+        except NotFound:
+            abort(404, _('Resource not found'))
+        except NotAuthorized:
+            abort(401, _('Unauthorized to read resource %s') % id)
+        return render('dataviewer/{type}.html'.format(type=preview_type))

ckan/lib/cli.py

@@ -4,6 +4,9 @@
 import logging
 from pprint import pprint
 import re
+import ckan.include.rjsmin as rjsmin
+import ckan.include.rcssmin as rcssmin
+import ckan.lib.fanstatic_resources as fanstatic_resources
 
 import paste.script
 from paste.registry import Registry
@@ -13,6 +16,35 @@
 #   i.e. do the imports in methods, after _load_config is called.
 #   Otherwise loggers get disabled.
 
+
+def parse_db_config(config_key='sqlalchemy.url'):
+    ''' Takes a config key for a database connection url and parses it into
+    a dictionary. Expects a url like:
+
+    'postgres://tester:pass@localhost/ckantest3'
+    '''
+    from pylons import config
+    url = config[config_key]
+    regex = [
+        '^\s*(?P<db_type>\w*)',
+        '://',
+        '(?P<db_user>[^:]*)',
+        ':?',
+        '(?P<db_pass>[^@]*)',
+        '@',
+        '(?P<db_host>[^/:]*)',
+        ':?',
+        '(?P<db_port>[^/]*)',
+        '/',
+        '(?P<db_name>[\w.-]*)'
+    ]
+    db_details_match = re.match(''.join(regex), url)
+    if not db_details_match:
+        raise Exception('Could not extract db details from url: %r' % url)
+    db_details = db_details_match.groupdict()
+    return db_details
+
+
 class MockTranslator(object):
     def gettext(self, value):
         return value
@@ -134,14 +166,7 @@ def command(self):
             sys.exit(1)
 
     def _get_db_config(self):
-        from pylons import config
-        url = config['sqlalchemy.url']
-        # e.g. 'postgres://tester:pass@localhost/ckantest3'
-        db_details_match = re.match('^\s*(?P<db_type>\w*)://(?P<db_user>[^:]*):?(?P<db_pass>[^@]*)@(?P<db_host>[^/:]*):?(?P<db_port>[^/]*)/(?P<db_name>[\w.-]*)', url)
-        if not db_details_match:
-            raise Exception('Could not extract db details from url: %r' % url)
-        db_details = db_details_match.groupdict()
-        return db_details
+        return parse_db_config()
 
     def _get_postgres_cmd(self, command):
         self.db_details = self._get_db_config()
@@ -1642,3 +1667,66 @@ def mangle_po(self):
         po.save(out_po)
         po.save_as_mofile(out_mo)
         print 'zh_TW has been mangled'
+
+
+class MinifyCommand(CkanCommand):
+    '''Create minified versions of the given Javascript and CSS files.
+
+    Usage:
+
+        paster minify [FILE|DIRECTORY] ...
+
+    for example:
+
+        paster minify ckan/public/base
+        paster minify ckan/public/base/css/*.css
+        paster minify ckan/public/base/css/red.css
+    '''
+    summary = __doc__.split('\n')[0]
+    usage = __doc__
+    min_args = 1
+
+    def command(self):
+        self._load_config()
+        for base_path in self.args:
+            if os.path.isfile(base_path):
+                self.minify_file(base_path)
+            elif os.path.isdir(base_path):
+                for root, dirs, files in os.walk(base_path):
+                    for filename in files:
+                        path = os.path.join(root, filename)
+                        self.minify_file(path)
+            else:
+                # Path is neither a file or a dir?
+                continue
+
+    def minify_file(self, path):
+        '''Create the minified version of the given file.
+
+        If the file is not a .js or .css file (e.g. it's a .min.js or .min.css
+        file, or it's some other type of file entirely) it will not be
+        minifed.
+
+        :param path: The path to the .js or .css file to minify
+
+        '''
+        path_only, extension = os.path.splitext(path)
+
+        if path_only.endswith('.min'):
+            # This is already a minified file.
+            return
+
+        if extension not in ('.css', '.js'):
+            # This is not a js or css file.
+            return
+
+        path_min = fanstatic_resources.min_path(path)
+
+        source = open(path, 'r').read()
+        f = open(path_min, 'w')
+        if path.endswith('.css'):
+            f.write(rcssmin.cssmin(source))
+        elif path.endswith('.js'):
+            f.write(rjsmin.jsmin(source))
+        f.close()
+        print "Minified file '{0}'".format(path)

ckan/lib/fanstatic_resources.py

@@ -9,16 +9,22 @@
 # This imports patches fanstatic
 import ckan.lib.fanstatic_extensions as fanstatic_extensions
 
-from ckan.include.rjsmin import jsmin
-from ckan.include.rcssmin import cssmin
-
 log = logging.getLogger(__name__)
 
 
+def min_path(path):
+    '''Return the .min.* filename for the given .js or .css file.
+
+    For example moo.js -> moo.min.js
+
+    '''
+    path, ext = os.path.splitext(path)
+    return path + '.min' + ext
+
+
 def create_library(name, path, depend_base=True):
     ''' Creates a fanstatic library `name` with the contents of a
-    directory `path` using resource.config if found. Files are minified
-    if needed. '''
+    directory `path` using resource.config if found.'''
 
     def get_resource(lib_name, resource_name):
         ''' Attempt to get the resource from the current lib or if not try
@@ -29,31 +35,6 @@ def get_resource(lib_name, resource_name):
             res = getattr(module, '%s' % resource_name)
         return res
 
-    def min_path(path):
-        ''' return the .min filename eg moo.js -> moo.min.js '''
-        if path.endswith('.js'):
-            return path[:-3] + '.min.js'
-        if path.endswith('.css'):
-            return path[:-4] + '.min.css'
-
-    def minify(filename, resource_path, min_function):
-        ''' Minify file path using min_function. '''
-        # if the minified file was modified after the source file we can
-        # assume that it is up-to-date
-        path = os.path.join(resource_path, filename)
-        path_min = min_path(path)
-        op = os.path
-        if op.exists(path_min) and op.getmtime(path) < op.getmtime(path_min):
-            return
-        source = open(path, 'r').read()
-        try:
-            f = open(path_min, 'w')
-            f.write(min_function(source))
-            f.close()
-            log.debug('minified %s' % path)
-        except IOError:
-            pass
-
     def create_resource(path, lib_name, count, inline=False):
         ''' create the fanstatic Resource '''
         renderer = None
@@ -186,12 +167,10 @@ def create_resource(path, lib_name, count, inline=False):
             if rel_path:
                 rel_path = rel_path[1:]
             filepath = os.path.join(rel_path, f)
-            if f.endswith('.js') and not f.endswith('.min.js'):
-                minify(f, dirname, jsmin)
-                resource_list.append(filepath)
-            if f.endswith('.css') and not f.endswith('.min.css'):
-                minify(f, dirname, cssmin)
-                resource_list.append(filepath)
+            filename_only, extension = os.path.splitext(f)
+            if extension in ('.css', '.js') and (
+                not filename_only.endswith('.min')):
+              resource_list.append(filepath)
 
     # if groups are defined make sure the order supplied there is honored
     for group in groups:
@@ -248,10 +227,10 @@ def create_resource(path, lib_name, count, inline=False):
 
 create_library('vendor', os.path.join(base_path, 'vendor'), depend_base=False)
 
-create_library('datapreview', os.path.join(base_path, 'datapreview'),
+create_library('base', os.path.join(base_path, 'javascript'),
                depend_base=False)
 
-create_library('base', os.path.join(base_path, 'javascript'),
+create_library('datapreview', os.path.join(base_path, 'datapreview'),
                depend_base=False)
 
 create_library('css', os.path.join(base_path, 'css'), depend_base=False)