Merge branch 'master' of github.com:okfn/ckan into 2257-clean-helper-…

…functions

Conflicts:
	ckan/lib/helpers.py

ckan/config/deployment.ini_tmpl

@@ -4,7 +4,16 @@
 # The %(here)s variable will be replaced with the parent directory of this file
 #
 [DEFAULT]
-debug = true
+
+# Change debug to true when doing CKAN development, it enables Pylons'
+# interactive debugging tool, makes Fanstatic serve unminified JS and CSS
+# files, and enables CKAN templates' debugging features.
+#   
+# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PRODUCTION ENVIRONMENT*
+# Debug mode will enable the interactive debugging tool, allowing ANYONE to
+# execute malicious code after an exception is raised.
+debug = false
+
 email_to = you@yourdomain.com
 smtp_server = localhost
 error_email_from = paste@localhost
@@ -32,18 +41,18 @@ ckan.plugins = stats
 #beaker.cache.data_dir = %(here)s/data/cache
 #beaker.session.data_dir = %(here)s/data/sessions
 
-# WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*
-# Debug mode will enable the interactive debugging tool, allowing ANYONE to
-# execute malicious code after an exception is raised.
-set debug = false
-
 # Specify the database for SQLAlchemy to use:
 # * Postgres is currently required for a production CKAN deployment
 # * Sqlite (memory or file) can be used as a quick alternative for testing
 sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckantest
 #sqlalchemy.url = sqlite:///
 #sqlalchemy.url = sqlite:///%(here)s/somedb.db
 
+## Datastore
+## Uncommment to set the datastore urls
+#ckan.datastore.write_url = postgresql://ckanuser:pass@localhost/ckantest
+#ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/ckantest
+
 # repoze.who config
 who.config_file = %(here)s/who.ini
 who.log_level = warning
@@ -226,10 +235,6 @@ ckan.feeds.author_link =
 #ofs.aws_access_key_id = ....
 #ofs.aws_secret_access_key = ....
 
-## Webstore
-## Uncommment to enable datastore
-# ckan.datastore.enabled = 1
-
 ## ===================================
 ## Extensions
 

ckan/config/environment.py

@@ -125,7 +125,7 @@ def find_controller(self, controller):
     paths = dict(root=root,
                  controllers=os.path.join(root, 'controllers'),
                  static_files=os.path.join(root, 'public'),
-                 templates=[os.path.join(root, 'templates')])
+                 templates=[])
 
     # Initialize config with the basic options
 
@@ -186,15 +186,13 @@ def find_controller(self, controller):
     ## redo template setup to use genshi.search_path
     ## (so remove std template setup)
     legacy_templates_path = os.path.join(root, 'templates_legacy')
+    jinja2_templates_path = os.path.join(root, 'templates')
     if asbool(config.get('ckan.legacy_templates', 'no')):
-        template_paths = [legacy_templates_path]
-        # if we are testing allow new templates
-        if asbool(config.get('ckan.enable_testing', 'false')):
-            jinja2_templates_path = os.path.join(root, 'templates')
-            template_paths.append(jinja2_templates_path)
+        # We want the new template path for extra snippets like the
+        # dataviewer and also for some testing stuff
+        template_paths = [legacy_templates_path, jinja2_templates_path]
     else:
-        template_paths = [paths['templates'][0]]
-        template_paths.append(legacy_templates_path)
+        template_paths = [jinja2_templates_path, legacy_templates_path]
 
     extra_template_paths = config.get('extra_template_paths', '')
     if extra_template_paths:

ckan/config/middleware.py

@@ -26,6 +26,7 @@
 from ckan.config.environment import load_environment
 import ckan.lib.app_globals as app_globals
 
+
 def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     """Create a Pylons WSGI application and return it
 
@@ -64,30 +65,29 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     app = RoutesMiddleware(app, config['routes.map'])
     app = SessionMiddleware(app, config)
     app = CacheMiddleware(app, config)
-    
+
     # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
     #app = QueueLogMiddleware(app)
-    
+
     # Fanstatic
     if asbool(config.get('debug', False)):
         fanstatic_config = {
-            'versioning' : True,
-            'recompute_hashes' : True,
-            'minified' : False,
-            'bottom' : True,
-            'bundle' : False,
+            'versioning': True,
+            'recompute_hashes': True,
+            'minified': False,
+            'bottom': True,
+            'bundle': False,
         }
     else:
         fanstatic_config = {
-            'versioning' : True,
-            'recompute_hashes' : False,
-            'minified' : True,
-            'bottom' : True,
-            'bundle' : True,
+            'versioning': True,
+            'recompute_hashes': False,
+            'minified': True,
+            'bottom': True,
+            'bundle': True,
         }
     app = Fanstatic(app, **fanstatic_config)
 
-
     if asbool(full_stack):
         # Handle Python exceptions
         app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
@@ -98,15 +98,15 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
             app = StatusCodeRedirect(app, [400, 404])
         else:
             app = StatusCodeRedirect(app, [400, 404, 500])
-    
+
     # Initialize repoze.who
     who_parser = WhoConfig(global_conf['here'])
     who_parser.parse(open(app_conf['who.config_file']))
 
     if asbool(config.get('openid_enabled', 'true')):
         from repoze.who.plugins.openid.identification import OpenIdIdentificationPlugin
         # Monkey patches for repoze.who.openid
-        # Fixes #1659 - enable log-out when CKAN mounted at non-root URL 
+        # Fixes #1659 - enable log-out when CKAN mounted at non-root URL
         from ckan.lib import repoze_patch
         OpenIdIdentificationPlugin.identify = repoze_patch.identify
         OpenIdIdentificationPlugin.redirect_to_logged_in = repoze_patch.redirect_to_logged_in
@@ -117,7 +117,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
                 not isinstance(i, OpenIdIdentificationPlugin)]
         who_parser.challengers = [i for i in who_parser.challengers if \
                 not isinstance(i, OpenIdIdentificationPlugin)]
-    
+
     app = PluggableAuthenticationMiddleware(app,
                 who_parser.identifiers,
                 who_parser.authenticators,
@@ -126,10 +126,10 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
                 who_parser.request_classifier,
                 who_parser.challenge_decider,
                 logging.getLogger('repoze.who'),
-                logging.WARN, # ignored
+                logging.WARN,  # ignored
                 who_parser.remote_user_key,
            )
-    
+
     # Establish the Registry for this application
     app = RegistryManager(app)
 
@@ -152,7 +152,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
                     StaticURLParser(public_path.strip(),
                         cache_max_age=static_max_age)
                 )
-        app = Cascade(extra_static_parsers+static_parsers)
+        app = Cascade(extra_static_parsers + static_parsers)
 
     # Page cache
     if asbool(config.get('ckan.page_cache_enabled')):
@@ -164,6 +164,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
 
     return app
 
+
 class I18nMiddleware(object):
     """I18n Middleware selects the language based on the url
     eg /fr/home is French"""
@@ -198,7 +199,7 @@ def __call__(self, environ, start_response):
             # Current application url
             path_info = environ['PATH_INFO']
             # sort out weird encodings
-            path_info = '/'.join(urllib.quote(pce,'') for pce in path_info.split('/'))
+            path_info = '/'.join(urllib.quote(pce, '') for pce in path_info.split('/'))
 
             qs = environ.get('QUERY_STRING')
 
@@ -317,7 +318,6 @@ def __init__(self, app, config):
         self.app = app
         self.engine = sa.create_engine(config.get('sqlalchemy.url'))
 
-
     def __call__(self, environ, start_response):
         path = environ['PATH_INFO']
         if path == '/_tracking':

ckan/config/routing.py

@@ -213,6 +213,8 @@ def make_map():
                   action='resource_embedded_dataviewer')
         m.connect('/dataset/{id}/resource/{resource_id}/viewer',
                   action='resource_embedded_dataviewer', width="960", height="800")
+        m.connect('/dataset/{id}/resource/{resource_id}/preview/{preview_type}',
+                  action='resource_datapreview')
 
     # group
     map.redirect('/groups', '/group')

ckan/controllers/group.py

@@ -439,7 +439,8 @@ def history(self, id):
 
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author,
-                   'schema': self._form_to_db_schema()}
+                   'schema': self._form_to_db_schema(),
+                   'extras_as_string': True}
         data_dict = {'id': id}
         try:
             c.group_dict = get_action('group_show')(context, data_dict)

ckan/controllers/package.py

@@ -310,7 +310,6 @@ def read(self, id, format='html'):
         try:
             c.pkg_dict = get_action('package_show')(context, data_dict)
             c.pkg = context['package']
-            c.resources_json = json.dumps(c.pkg_dict.get('resources', []))
         except NotFound:
             abort(404, _('Dataset not found'))
         except NotAuthorized:
@@ -1146,7 +1145,6 @@ def resource_read(self, id, resource_id):
             c.package = get_action('package_show')(context, {'id': id})
             # required for nav menu
             c.pkg = context['package']
-            c.resource_json = json.dumps(c.resource)
             c.pkg_dict = c.package
         except NotFound:
             abort(404, _('Resource not found'))
@@ -1159,8 +1157,9 @@ def resource_read(self, id, resource_id):
                 get_license_register()[license_id].isopen()
         except KeyError:
             c.package['isopen'] = False
-        c.datastore_api = h.url_for('datastore_read', id=c.resource.get('id'),
-                                    qualified=True)
+
+        # TODO: find a nicer way of doing this
+        c.datastore_api = '%s/api/action' % config.get('ckan.site_url','').rstrip('/')
 
         c.related_count = c.pkg.related_count
         return render('package/resource_read.html')
@@ -1280,3 +1279,21 @@ def _parse_recline_state(self, params):
                     not k.endswith(recline_state['currentView']):
                 recline_state.pop(k)
         return recline_state
+
+    def resource_datapreview(self, id, resource_id, preview_type):
+        '''
+        Embeded page for a resource data-preview.
+        '''
+        context = {'model': model, 'session': model.Session,
+                   'user': c.user or c.author}
+
+        try:
+            c.resource = get_action('resource_show')(context,
+                                                     {'id': resource_id})
+            c.package = get_action('package_show')(context, {'id': id})
+            c.resource_json = json.dumps(c.resource)
+        except NotFound:
+            abort(404, _('Resource not found'))
+        except NotAuthorized:
+            abort(401, _('Unauthorized to read resource %s') % id)
+        return render('dataviewer/{type}.html'.format(type=preview_type))

ckan/controllers/storage.py

@@ -262,8 +262,9 @@ def get_metadata(self, label):
         if storage_backend in ['google', 's3']:
             if not label.startswith("/"):
                 label = "/" + label
-            url = "https://%s/%s%s" % (self.ofs.conn.server_name(),
-                                       bucket, label)
+            url = "https://%s%s" % (
+                self.ofs.conn.calling_format.build_host(
+                    self.ofs.conn.server_name(), bucket), label)
         else:
             url = h.url_for('storage_file',
                             label=label,