Merged master

ckan/controllers/package.py

@@ -619,7 +619,6 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
             del data['id']
 
             context = {'model': model, 'session': model.Session,
-                       'api_version': 3,
                        'user': c.user or c.author,
                        'extras_as_string': True}
 
@@ -705,7 +704,6 @@ def new_metadata(self, id, data=None, errors=None, error_summary=None):
             # we don't want to include save as it is part of the form
             del data['save']
             context = {'model': model, 'session': model.Session,
-                       'api_version': 3,
                        'user': c.user or c.author,
                        'extras_as_string': True}
             data_dict = get_action('package_show')(context, {'id': id})
@@ -989,7 +987,6 @@ def _save_edit(self, name_or_id, context, package_type=None):
             data_dict = clean_dict(unflatten(
                 tuplize_dict(parse_params(request.POST))))
             if '_ckan_phase' in data_dict:
-                context['api_version'] = 3
                 # we allow partial updates to not destroy existing resources
                 context['allow_partial_update'] = True
                 data_dict['tags'] = self._tag_string_to_list(

ckan/controllers/related.py

@@ -61,7 +61,7 @@ def pager_url(q=None, page=None):
             page=page,
             url=pager_url,
             item_count=query.count(),
-            items_per_page=8
+            items_per_page=9
         )
 
         c.filters = dict(params_nopage)

ckan/lib/activity_streams.py

@@ -13,29 +13,29 @@
 # etc.
 
 def get_snippet_actor(activity, detail):
-    return literal('''<span class="actor" data-module="popover-context" data-module-type="user" data-module-id="%s">%s</span>'''
-        % (activity['user_id'], h.linked_user(activity['user_id'], 0, 30))
+    return literal('''<span class="actor">%s</span>'''
+        % (h.linked_user(activity['user_id'], 0, 30))
         )
 
 def get_snippet_user(activity, detail):
-    return literal('''<span data-module="popover-context" data-module-type="user" data-module-id="%s">%s</span>'''
-        % (activity['object_id'], h.linked_user(activity['object_id'], 0, 20))
+    return literal('''<span>%s</span>'''
+        % (h.linked_user(activity['object_id'], 0, 20))
         )
 
 def get_snippet_dataset(activity, detail):
     data = activity['data']
     link = h.dataset_link(data.get('package') or data.get('dataset'))
-    return literal('''<span data-module="popover-context" data-module-type="dataset" data-module-id="%s">%s</span>'''
-        % (activity['object_id'], link)
+    return literal('''<span>%s</span>'''
+        % (link)
         )
 
 def get_snippet_tag(activity, detail):
     return h.tag_link(detail['data']['tag'])
 
 def get_snippet_group(activity, detail):
     link = h.group_link(activity['data']['group'])
-    return literal('''<span data-module="popover-context" data-module-type="group" data-module-id="%s">%s</span>'''
-        % (activity['object_id'], link)
+    return literal('''<span>%s</span>'''
+        % (link)
         )
 
 def get_snippet_organization(activity, detail):

ckan/lib/app_globals.py

@@ -3,10 +3,12 @@
 import logging
 import time
 from threading import Lock
+import re
 
 from paste.deploy.converters import asbool
 from pylons import config
 
+import ckan
 import ckan.model as model
 
 log = logging.getLogger(__name__)
@@ -49,7 +51,7 @@
                       'type': 'split',
                       'name': 'facets'},
     'package_hide_extras': {'type': 'split'},
-    'plugins': {'type': 'split'},
+    'ckan.plugins': {'type': 'split'},
 
     # bool
     'openid_enabled': {'default': 'true', 'type' : 'bool'},
@@ -172,6 +174,10 @@ def _check_uptodate(self):
                 self._mutex.release()
 
     def _init(self):
+
+        self.ckan_version = ckan.__version__
+        self.ckan_base_version = re.sub('[^0-9\.]', '', self.ckan_version)
+
         # process the config_details to set globals
         for name, options in config_details.items():
             if 'name' in options:

ckan/lib/cli.py

@@ -1756,13 +1756,16 @@ class MinifyCommand(CkanCommand):
     usage = __doc__
     min_args = 1
 
+    exclude_dirs = ['vendor']
+
     def command(self):
         self._load_config()
         for base_path in self.args:
             if os.path.isfile(base_path):
                 self.minify_file(base_path)
             elif os.path.isdir(base_path):
                 for root, dirs, files in os.walk(base_path):
+                    dirs[:] = [d for d in dirs if not d in self.exclude_dirs]
                     for filename in files:
                         path = os.path.join(root, filename)
                         self.minify_file(path)
@@ -1800,3 +1803,122 @@ def minify_file(self, path):
             f.write(rjsmin.jsmin(source))
         f.close()
         print "Minified file '{0}'".format(path)
+
+
+class LessCommand(CkanCommand):
+    '''Compile all root less documents into their CSS counterparts
+
+    Usage:
+
+        paster less
+
+    '''
+    summary = __doc__.split('\n')[0]
+    usage = __doc__
+    min_args = 0
+
+    def command(self):
+        self.less()
+
+    custom_css = {
+        'fuchsia': '''
+            @layoutLinkColor: #b509b5;
+            @mastheadBackgroundColorStart: #dc0bdc;
+            @mastheadBackgroundColorEnd: #f31df3;
+            @btnPrimaryBackground: #f544f5;
+            @btnPrimaryBackgroundHighlight: #f76bf7;
+            ''',
+
+        'green': '''
+            @layoutLinkColor: #045b04;
+            @mastheadBackgroundColorStart: #068106;
+            @mastheadBackgroundColorEnd: #08a808;
+            @btnPrimaryBackground: #0acf0a;
+            @btnPrimaryBackgroundHighlight: #10f210
+            ''',
+
+        'red': '''
+            @layoutLinkColor: #b50909;
+            @mastheadBackgroundColorStart: #dc0b0b;
+            @mastheadBackgroundColorEnd: #f31d1d;
+            @btnPrimaryBackground: #f54444;
+            @btnPrimaryBackgroundHighlight: #f76b6b;
+            ''',
+
+        'maroon': '''
+            @layoutLinkColor: #5b0404;
+            @mastheadBackgroundColorStart: #810606;
+            @mastheadBackgroundColorEnd: #a80808;
+            @btnPrimaryBackground: #cf0a0a;
+            @btnPrimaryBackgroundHighlight: #f21010;
+            ''',
+    }
+    def less(self):
+        ''' Compile less files '''
+        import subprocess
+        command = 'npm bin'
+        process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+        output = process.communicate()
+        directory = output[0].strip()
+        less_bin = os.path.join(directory, 'lessc')
+
+        root = os.path.join(os.path.dirname(__file__), '..', 'public', 'base')
+        root = os.path.abspath(root)
+        custom_less = os.path.join(root, 'less', 'custom.less')
+        for color in self.custom_css:
+            f = open(custom_less, 'w')
+            f.write(self.custom_css[color])
+            f.close()
+            self.compile_less(root, less_bin, color)
+        f = open(custom_less, 'w')
+        f.write('// This file is needed in order for ./bin/less to compile in less 1.3.1+\n')
+        f.close()
+        self.compile_less(root, less_bin, 'main')
+
+
+
+    def compile_less(self, root, less_bin, color):
+        print 'compile %s.css' % color
+        import subprocess
+        main_less = os.path.join(root, 'less', 'main.less')
+        main_css = os.path.join(root, 'css', '%s.css' % color)
+
+        command = '%s %s %s' % (less_bin, main_less, main_css)
+
+        process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+        output = process.communicate()
+
+
+
+class FrontEndBuildCommand(CkanCommand):
+    ''' Creates and minifies css and JavaScript files
+
+    Usage:
+
+        paster front-end-build
+    '''
+
+    summary = __doc__.split('\n')[0]
+    usage = __doc__
+    min_args = 0
+
+    def command(self):
+        self._load_config()
+
+        # Less css
+        cmd = LessCommand('less')
+        cmd.command()
+
+        # js translation strings
+        cmd = TranslationsCommand('trans')
+        cmd.options = self.options
+        cmd.args = ('js',)
+        cmd.command()
+
+        # minification
+        cmd = MinifyCommand('minify')
+        cmd.options = self.options
+        root = os.path.join(os.path.dirname(__file__), '..', 'public', 'base')
+        root = os.path.abspath(root)
+        cmd.args = (root,)
+        cmd.command()

ckan/lib/navl/dictization_functions.py

@@ -226,15 +226,14 @@ def validate(data, schema, context=None):
 
     flattened = flatten_dict(data)
     converted_data, errors = _validate(flattened, schema, context)
+    converted_data = unflatten(converted_data)
 
     # check config for partial update fix option
-    if config.get('ckan.fix_partial_updates', False):
+    if config.get('ckan.fix_partial_updates', True):
         # repopulate the empty lists
         for key in empty_lists:
             if key not in converted_data:
-                converted_data[(key,)] = []
-
-    converted_data = unflatten(converted_data)
+                converted_data[key] = []
 
     errors_unflattened = unflatten(errors)
 

ckan/lib/plugins.py

@@ -271,7 +271,9 @@ def check_data_dict(self, data_dict, schema=None):
         # Resources might not exist yet (eg. Add Dataset)
         surplus_keys_schema = ['__extras', '__junk', 'state', 'groups',
                                'extras_validation', 'save', 'return_to',
-                               'resources', 'type', 'owner_org']
+                               'resources', 'type', 'owner_org',
+                               'log_message', 'tag_string', 'tags',
+                               'url', 'version', 'extras']
 
         if not schema:
             schema = self.form_to_db_schema()

ckan/logic/schema.py

@@ -186,31 +186,81 @@ def form_to_db_package_schema():
 
     ##changes
     schema.pop("id")
-    schema.pop('tags')
     schema.pop('relationships_as_object')
     schema.pop('revision_id')
     schema.pop('relationships_as_subject')
+
     return schema
 
 def db_to_form_package_schema():
     schema = default_package_schema()
-    # Workaround a bug in CKAN's convert_from_tags() function.
-    # TODO: Fix this issue in convert_from_tags().
+
     schema.update({
         'tags': {
             '__extras': [ckan.lib.navl.validators.keep_extras,
                 ckan.logic.converters.free_tags_only]
             },
         })
-    # Workaround a bug in CKAN.
-    # TODO: Fix this elsewhere so we don't need to workaround it here.
+
+    # Add several keys to the 'resources' subschema so they don't get stripped
+    # from the resource dicts by validation.
     schema['resources'].update({
         'created': [ckan.lib.navl.validators.ignore_missing],
         'position': [not_empty],
         'last_modified': [ckan.lib.navl.validators.ignore_missing],
         'cache_last_updated': [ckan.lib.navl.validators.ignore_missing],
         'webstore_last_updated': [ckan.lib.navl.validators.ignore_missing],
+        'revision_timestamp': [],
+        'resource_group_id': [],
+        'cache_last_updated': [],
+        'webstore_last_updated': [],
+        'size': [],
+        'state': [],
+        'last_modified': [],
+        'mimetype': [],
+        'cache_url': [],
+        'name': [],
+        'webstore_url': [],
+        'mimetype_inner': [],
+        'resource_type': [],
     })
+
+    schema.update({
+        'state': [ckan.lib.navl.validators.ignore_missing],
+        'isopen': [ignore_missing],
+        'license_url': [ignore_missing],
+        })
+
+    schema['groups'].update({
+        'description': [ignore_missing],
+        })
+
+    # Remove validators for several keys from the schema so validation doesn't
+    # strip the keys from the package dicts if the values are 'missing' (i.e.
+    # None).
+    schema['author'] = []
+    schema['author_email'] = []
+    schema['maintainer'] = []
+    schema['maintainer_email'] = []
+    schema['license_id'] = []
+    schema['notes'] = []
+    schema['url'] = []
+    schema['version'] = []
+
+    # Add several keys that are missing from default_package_schema(), so
+    # validation doesn't strip the keys from the package dicts.
+    schema['metadata_created'] = []
+    schema['metadata_modified'] = []
+    schema['num_resources'] = []
+    schema['num_tags'] = []
+    schema['organization'] = []
+    schema['owner_org'] = []
+    schema['private'] = []
+    schema['revision_id'] = []
+    schema['revision_timestamp'] = []
+    schema['tracking_summary'] = []
+    schema['license_title'] = []
+
     return schema
 
 def default_group_schema():

ckan/plugins/core.py

@@ -11,7 +11,7 @@
 from pyutilib.component.core import SingletonPlugin as _pca_SingletonPlugin
 from pyutilib.component.core import Plugin as _pca_Plugin
 
-from ckan.plugins.interfaces import IPluginObserver
+from ckan.plugins.interfaces import IPluginObserver, IGenshiStreamFilter
 
 __all__ = [
     'PluginImplementations', 'implements',
@@ -123,6 +123,10 @@ def load(plugin):
     service.activate()
     for observer_plugin in observers:
         observer_plugin.after_load(service)
+
+    if IGenshiStreamFilter in service.__interfaces__:
+       log.warn("Plugin '%s' is using deprecated interface IGenshiStreamFilter" % plugin)
+
     return service