diff --git a/ckan/ckan_nose_plugin.py b/ckan/ckan_nose_plugin.py index 7018e31c23c..bc995dbbdb3 100644 --- a/ckan/ckan_nose_plugin.py +++ b/ckan/ckan_nose_plugin.py @@ -30,7 +30,6 @@ def startContext(self, ctx): # init_db is run at the start of every class because # when you use an in-memory sqlite db, it appears that # the db is destroyed after every test when you Session.Remove(). - model.repo.init_db() ## This is to make sure the configuration is run again. ## Plugins use configure to make their own tables and they @@ -40,6 +39,7 @@ def startContext(self, ctx): for plugin in PluginImplementations(IConfigurable): plugin.configure(config) + model.repo.init_db() def options(self, parser, env): parser.add_option( diff --git a/ckan/config/environment.py b/ckan/config/environment.py index ff3bc45fbb5..8d42f4bedef 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -30,10 +30,14 @@ class _Helpers(object): def __init__(self, helpers, restrict=True): functions = {} allowed = helpers.__allowed_functions__ + # list of functions due to be depreciated + self.depreciated = [] for helper in dir(helpers): - if restrict and (helper not in allowed): - continue + if helper not in allowed: + self.depreciated.append(helper) + if restrict: + continue functions[helper] = getattr(helpers, helper) self.functions = functions @@ -46,6 +50,8 @@ def __init__(self, helpers, restrict=True): raise Exception('overwritting extra helper %s' % helper) extra_helpers.append(helper) functions[helper] = helpers[helper] + # logging + self.log = logging.getLogger('ckan.helpers') @classmethod def null_function(cls, *args, **kw): @@ -57,10 +63,21 @@ def null_function(cls, *args, **kw): def __getattr__(self, name): ''' return the function/object requested ''' if name in self.functions: + if name in self.depreciated: + msg = 'Template helper function `%s` is depriciated' % name + self.log.warn(msg) return self.functions[name] else: - log = logging.getLogger('ckan.helpers') - log.critical('Helper function `%s` could not be found (missing extension?)' % name) + if name in self.depreciated: + msg = 'Template helper function `%s` is not available ' \ + 'as it has been depriciated.\nYou can enable it ' \ + 'by setting ckan.restrict_template_vars = true ' \ + 'in your .ini file.' % name + self.log.critical(msg) + else: + msg = 'Helper function `%s` could not be found\n ' \ + '(are you missing an extension?)' % name + self.log.critical(msg) return self.null_function @@ -171,9 +188,8 @@ def template_loaded(template): ckan_db = os.environ.get('CKAN_DB') if ckan_db: - engine = sqlalchemy.create_engine(ckan_db) - else: - engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.') + config['sqlalchemy.url'] = ckan_db + engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.') if not model.meta.engine: model.init_model(engine) diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index c07948a51a8..5d2be57b77e 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -1,8 +1,11 @@ """Pylons middleware initialization""" import urllib -import logging +import urllib2 +import logging import json +import hashlib +import sqlalchemy as sa from beaker.middleware import CacheMiddleware, SessionMiddleware from paste.cascade import Cascade from paste.registry import RegistryManager @@ -14,6 +17,7 @@ from routes.middleware import RoutesMiddleware from repoze.who.config import WhoConfig from repoze.who.middleware import PluggableAuthenticationMiddleware + from ckan.plugins import PluginImplementations from ckan.plugins.interfaces import IMiddleware from ckan.lib.i18n import get_locales @@ -130,6 +134,8 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf): if asbool(config.get('ckan.page_cache_enabled')): app = PageCacheMiddleware(app, config) + # Tracking add config option + app = TrackingMiddleware(app, config) return app class I18nMiddleware(object): @@ -277,3 +283,40 @@ def _start_response(status, response_headers, exc_info=None): pipe.rpush(key, page_string) pipe.execute() return page + + +class TrackingMiddleware(object): + + def __init__(self, app, config): + self.app = app + self.engine = sa.create_engine(config.get('sqlalchemy.url')) + + + def __call__(self, environ, start_response): + path = environ['PATH_INFO'] + if path == '/_tracking': + # do the tracking + # get the post data + payload = environ['wsgi.input'].read() + parts = payload.split('&') + data = {} + for part in parts: + k, v = part.split('=') + data[k] = urllib2.unquote(v).decode("utf8") + start_response('200 OK', [('Content-Type', 'text/html')]) + # we want a unique anonomized key for each user so that we do + # not count multiple clicks from the same user. + key = ''.join([ + environ['HTTP_USER_AGENT'], + environ['REMOTE_ADDR'], + environ['HTTP_ACCEPT_LANGUAGE'], + environ['HTTP_ACCEPT_ENCODING'], + ]) + key = hashlib.md5(key).hexdigest() + # store key/data here + sql = '''INSERT INTO tracking_raw + (user_key, url, tracking_type) + VALUES (%s, %s, %s)''' + self.engine.execute(sql, key, data.get('url'), data.get('type')) + return [] + return self.app(environ, start_response) diff --git a/ckan/config/routing.py b/ckan/config/routing.py index f3ae5b8b34c..6c03cb096aa 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -57,6 +57,7 @@ def make_map(): 'resource', 'tag', 'group', + 'related', 'authorizationgroup', 'revision', 'licenses', @@ -151,6 +152,11 @@ def make_map(): ##map.connect('/package/new', controller='package_formalchemy', action='new') ##map.connect('/package/edit/{id}', controller='package_formalchemy', action='edit') + with SubMapper(map, controller='related') as m: + m.connect('related_edit', '/related/{id}/edit', action='edit') + m.connect('related_list', '/dataset/{id}/related', action='list') + m.connect('related_read', '/dataset/{id}/related/{related_id}', action='read') + with SubMapper(map, controller='package') as m: m.connect('/dataset', action='search') m.connect('/dataset/{action}', @@ -183,6 +189,7 @@ def make_map(): m.connect('/dataset/{id}.{format}', action='read') m.connect('/dataset/{id}', action='read') m.connect('/dataset/{id}/resource/{resource_id}', action='resource_read') + m.connect('/dataset/{id}/resource/{resource_id}/embed', action='resource_embedded_dataviewer') # group map.redirect('/groups', '/group') @@ -210,7 +217,6 @@ def make_map(): register_package_plugins(map) register_group_plugins(map) - # authz group map.redirect('/authorizationgroups', '/authorizationgroup') map.redirect('/authorizationgroups/{url:.*}', '/authorizationgroup/{url}') diff --git a/ckan/config/solr/schema-1.4.xml b/ckan/config/solr/schema-1.4.xml new file mode 100644 index 00000000000..29cb4737287 --- /dev/null +++ b/ckan/config/solr/schema-1.4.xml @@ -0,0 +1,175 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +index_id +text + + + + + + + + + + + + + + + + + + + + diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index 33bb54a51ae..a587f3d824d 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -235,6 +235,7 @@ def list(self, ver=None, register=None, subregister=None, id=None): 'group': 'group_list', 'dataset': 'package_list', 'tag': 'tag_list', + 'related': 'related_list', 'licenses': 'licence_list', ('dataset', 'relationships'): 'package_relationships_list', ('dataset', 'revisions'): 'package_revision_list', @@ -261,6 +262,7 @@ def show(self, ver=None, register=None, subregister=None, id=None, id2=None): 'revision': 'revision_show', 'group': 'group_show_rest', 'tag': 'tag_show_rest', + 'related': 'related_show', 'dataset': 'package_show_rest', ('dataset', 'relationships'): 'package_relationships_list', } @@ -294,6 +296,7 @@ def create(self, ver=None, register=None, subregister=None, id=None, id2=None): 'group': 'group_create_rest', 'dataset': 'package_create_rest', 'rating': 'rating_create', + 'related': 'related_create', ('dataset', 'relationships'): 'package_relationship_create_rest', } for type in model.PackageRelationship.get_all_types(): @@ -393,6 +396,7 @@ def delete(self, ver=None, register=None, subregister=None, id=None, id2=None): action_map = { 'group': 'group_delete', 'dataset': 'package_delete', + 'related': 'related_delete', ('dataset', 'relationships'): 'package_relationship_delete_rest', } for type in model.PackageRelationship.get_all_types(): @@ -621,6 +625,7 @@ def convert_to_dict(user): def is_slug_valid(self): slug = request.params.get('slug') or '' slugtype = request.params.get('type') or '' + # TODO: We need plugins to be able to register new disallowed names disallowed = ['new', 'edit', 'search'] if slugtype==u'package': response_data = dict(valid=not bool(common.package_exists(slug) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 4e7cda3ef10..95a4597ad70 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -72,7 +72,7 @@ def index(self): group_type = self._guess_group_type() context = {'model': model, 'session': model.Session, - 'user': c.user or c.author} + 'user': c.user or c.author, 'for_view': True} data_dict = {'all_fields': True} @@ -194,6 +194,7 @@ def pager_url(q=None, page=None): items_per_page=limit ) c.facets = query['facets'] + c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Group search error: %r', se.args) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index e90d65889e1..ba12b869761 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -210,6 +210,7 @@ def pager_url(q=None, page=None): items_per_page=limit ) c.facets = query['facets'] + c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) @@ -295,6 +296,7 @@ def read(self, id, format='html'): # used by disqus plugin c.current_package_id = c.pkg.id + c.related_count = len(c.pkg.related) # Add the package's activity stream (already rendered to HTML) to the # template context for the package/read.html template to retrieve @@ -403,6 +405,8 @@ def history(self, id): ) feed.content_type = 'application/atom+xml' return feed.writeString('utf-8') + + c.related_count = len(c.pkg.related) return render( self._history_template(c.pkg_dict.get('type',package_type))) def new(self, data=None, errors=None, error_summary=None): @@ -658,6 +662,9 @@ def authz(self, id): roles = self._handle_update_of_authz(pkg) self._prepare_authz_info_for_render(roles) + + # c.related_count = len(pkg.related) + return render('package/authz.html') def autocomplete(self): @@ -744,3 +751,63 @@ def resource_read(self, id, resource_id): qualified=True) return render('package/resource_read.html') + def resource_embedded_dataviewer(self, id, resource_id): + """ + Embeded page for a read-only resource dataview. + """ + context = {'model': model, 'session': model.Session, + 'user': c.user or c.author} + + try: + c.resource = get_action('resource_show')(context, {'id': resource_id}) + c.package = get_action('package_show')(context, {'id': id}) + c.resource_json = json.dumps(c.resource) + + # double check that the resource belongs to the specified package + if not c.resource['id'] in [ r['id'] for r in c.package['resources'] ]: + raise NotFound + + except NotFound: + abort(404, _('Resource not found')) + except NotAuthorized: + abort(401, _('Unauthorized to read resource %s') % id) + + # Construct the recline state + state_version = int(request.params.get('state_version', '1')) + recline_state = self._parse_recline_state(request.params) + if recline_state is None: + abort(400, ('"state" parameter must be a valid recline state (version %d)' % state_version)) + + c.recline_state = json.dumps(recline_state) + + c.width = max(int(request.params.get('width', 500)), 100) + c.height = max(int(request.params.get('height', 500)), 100) + c.embedded = True + + return render('package/resource_embedded_dataviewer.html') + + def _parse_recline_state(self, params): + state_version = int(request.params.get('state_version', '1')) + if state_version != 1: + return None + + recline_state = {} + for k,v in request.params.items(): + try: + v = json.loads(v) + except ValueError: + pass + recline_state[k] = v + + recline_state.pop('width', None) + recline_state.pop('height', None) + recline_state['readOnly'] = True + + # Ensure only the currentView is available + if not recline_state.get('currentView', None): + recline_state['currentView'] = 'grid' # default to grid view if none specified + for k in recline_state.keys(): + if k.startswith('view-') and not k.endswith(recline_state['currentView']): + recline_state.pop(k) + return recline_state + diff --git a/ckan/controllers/related.py b/ckan/controllers/related.py new file mode 100644 index 00000000000..85042aaa8eb --- /dev/null +++ b/ckan/controllers/related.py @@ -0,0 +1,36 @@ + + +import ckan.model as model +import ckan.logic as logic +import ckan.lib.base as base +import ckan.lib.helpers as h + +c = base.c + +class RelatedController(base.BaseController): + + def list(self, id): + + context = {'model': model, 'session': model.Session, + 'user': c.user or c.author, 'extras_as_string': True, + 'for_view': True} + data_dict = {'id': id} + + try: + logic.check_access('package_show', context, data_dict) + except logic.NotAuthorized: + abort(401, _('Not authorized to see this page')) + + try: + c.pkg_dict = logic.get_action('package_show')(context, data_dict) + c.pkg = context['package'] + c.resources_json = h.json.dumps(c.pkg_dict.get('resources',[])) + except logic.NotFound: + abort(404, _('Dataset not found')) + except logic.NotAuthorized: + abort(401, _('Unauthorized to read package %s') % id) + + c.related_count = len(c.pkg.related) + + return base.render( "package/related_list.html") + diff --git a/ckan/controllers/tag.py b/ckan/controllers/tag.py index f623f05c643..06bccd768ca 100644 --- a/ckan/controllers/tag.py +++ b/ckan/controllers/tag.py @@ -25,9 +25,9 @@ def index(self): c.q = request.params.get('q', '') context = {'model': model, 'session': model.Session, - 'user': c.user or c.author} + 'user': c.user or c.author, 'for_view': True} - data_dict = {} + data_dict = {'all_fields': True} if c.q: page = int(request.params.get('page', 1)) @@ -58,7 +58,7 @@ def index(self): def read(self, id): context = {'model': model, 'session': model.Session, - 'user': c.user or c.author} + 'user': c.user or c.author, 'for_view': True} data_dict = {'id':id} try: diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index 0a26d59c0f3..d17df6fc410 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -83,7 +83,7 @@ def index(self): def read(self, id=None): context = {'model': model, - 'user': c.user or c.author} + 'user': c.user or c.author, 'for_view': True} data_dict = {'id':id, 'user_obj':c.userobj} try: @@ -383,6 +383,8 @@ def perform_reset(self, id): h.flash_error(u'%r'% e.error_dict) except ValueError, ve: h.flash_error(unicode(ve)) + + c.user_dict = user_dict return render('user/perform_reset.html') def _format_about(self, about): diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 075029c8947..ffb79a7c4fb 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -53,16 +53,19 @@ def render_snippet(template_name, **kw): comment tags added to show the template used. NOTE: unlike other render functions this takes a list of keywords instead of a dict for the extra template variables. ''' - output = render(template_name, extra_vars=kw) + # allow cache_force to be set in render function + cache_force = kw.pop('cache_force', None) + output = render(template_name, extra_vars=kw, cache_force=cache_force) output = '\n\n%s\n\n' % ( template_name, output, template_name) return literal(output) -def render_text(template_name, extra_vars=None): +def render_text(template_name, extra_vars=None, cache_force=None): ''' Helper function to render a genshi NewTextTemplate without having to pass the loader_class or method. ''' return render(template_name, extra_vars=extra_vars, + cache_force=cache_force, method='text', loader_class=NewTextTemplate) @@ -245,8 +248,8 @@ def __after__(self, action, **params): def _set_cors(self): response.headers['Access-Control-Allow-Origin'] = "*" - response.headers['Access-Control-Allow-Methods'] = "POST, PUT, GET, DELETE" - response.headers['Access-Control-Allow-Headers'] = "X-CKAN-API-KEY, Content-Type" + response.headers['Access-Control-Allow-Methods'] = "POST, PUT, GET, DELETE, OPTIONS" + response.headers['Access-Control-Allow-Headers'] = "X-CKAN-API-KEY, Authorization, Content-Type" def _get_user(self, reference): return model.User.by_name(reference) diff --git a/ckan/lib/celery_app.py b/ckan/lib/celery_app.py index cd87aa7a0eb..27d2951d10b 100644 --- a/ckan/lib/celery_app.py +++ b/ckan/lib/celery_app.py @@ -1,5 +1,6 @@ import ConfigParser import os +from pylons import config as pylons_config from pkg_resources import iter_entry_points #from celery.loaders.base import BaseLoader @@ -12,16 +13,22 @@ config = ConfigParser.ConfigParser() config_file = os.environ.get('CKAN_CONFIG') + if not config_file: config_file = os.path.join( os.path.dirname(os.path.abspath(__file__)), '../../development.ini') config.read(config_file) +sqlalchemy_url = pylons_config.get('sqlalchemy.url') +if not sqlalchemy_url: + sqlalchemy_url = config.get('app:main', 'sqlalchemy.url') + + default_config = dict( BROKER_BACKEND = 'sqlalchemy', - BROKER_HOST = config.get('app:main', 'sqlalchemy.url'), - CELERY_RESULT_DBURI = config.get('app:main', 'sqlalchemy.url'), + BROKER_HOST = sqlalchemy_url, + CELERY_RESULT_DBURI = sqlalchemy_url, CELERY_RESULT_BACKEND = 'database', CELERY_RESULT_SERIALIZER = 'json', CELERY_TASK_SERIALIZER = 'json', diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 6798cc48359..d11a1f450e1 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -1,4 +1,5 @@ import os +import datetime import sys import logging from pprint import pprint @@ -879,3 +880,207 @@ def clean(self, user_ratings=True): rating.purge() model.repo.commit_and_remove() +class Tracking(CkanCommand): + '''Update tracking statistics + + Usage: + tracking - update tracking stats + ''' + + summary = __doc__.split('\n')[0] + usage = __doc__ + max_args = 1 + min_args = 0 + + def command(self): + self._load_config() + import ckan.model as model + engine = model.meta.engine + + if len(self.args) == 1: + # Get summeries from specified date + start_date = datetime.datetime.strptime(self.args[0], '%Y-%m-%d') + else: + # No date given. See when we last have data for and get data + # from 2 days before then in case new data is available. + # If no date here then use 2010-01-01 as the start date + sql = '''SELECT tracking_date from tracking_summary + ORDER BY tracking_date DESC LIMIT 1;''' + result = engine.execute(sql).fetchall() + if result: + start_date = result[0]['tracking_date'] + start_date += datetime.timedelta(-2) + # convert date to datetime + combine = datetime.datetime.combine + start_date = combine(start_date, datetime.time(0)) + else: + start_date = datetime.datetime(2011, 1, 1) + end_date = datetime.datetime.now() + + while start_date < end_date: + stop_date = start_date + datetime.timedelta(1) + self.update_tracking(engine, start_date) + print 'tracking updated for %s' % start_date + start_date = stop_date + + def update_tracking(self, engine, summary_date): + PACKAGE_URL = '/dataset/' + # clear out existing data before adding new + sql = '''DELETE FROM tracking_summary + WHERE tracking_date='%s'; ''' % summary_date + engine.execute(sql) + + sql = '''SELECT DISTINCT url, user_key, + CAST(access_timestamp AS Date) AS tracking_date, + tracking_type INTO tracking_tmp + FROM tracking_raw + WHERE CAST(access_timestamp as Date)='%s'; + + INSERT INTO tracking_summary + (url, count, tracking_date, tracking_type) + SELECT url, count(user_key), tracking_date, tracking_type + FROM tracking_tmp + GROUP BY url, tracking_date, tracking_type; + + DROP TABLE tracking_tmp; + COMMIT;''' % summary_date + engine.execute(sql) + + # get ids for dataset urls + sql = '''UPDATE tracking_summary t + SET package_id = COALESCE( + (SELECT id FROM package p + WHERE t.url = %s || p.name) + ,'~~not~found~~') + WHERE t.package_id IS NULL + AND tracking_type = 'page';''' + engine.execute(sql, PACKAGE_URL) + + # update summary totals for resources + sql = '''UPDATE tracking_summary t1 + SET running_total = ( + SELECT sum(count) + FROM tracking_summary t2 + WHERE t1.url = t2.url + AND t2.tracking_date <= t1.tracking_date + ) + t1.count + ,recent_views = ( + SELECT sum(count) + FROM tracking_summary t2 + WHERE t1.url = t2.url + AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - 14 + ) + t1.count + WHERE t1.running_total = 0 AND tracking_type = 'resource';''' + engine.execute(sql) + + # update summary totals for pages + sql = '''UPDATE tracking_summary t1 + SET running_total = ( + SELECT sum(count) + FROM tracking_summary t2 + WHERE t1.package_id = t2.package_id + AND t2.tracking_date <= t1.tracking_date + ) + t1.count + ,recent_views = ( + SELECT sum(count) + FROM tracking_summary t2 + WHERE t1.package_id = t2.package_id + AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - 14 + ) + t1.count + WHERE t1.running_total = 0 AND tracking_type = 'page' + AND t1.package_id IS NOT NULL + AND t1.package_id != '~~not~found~~';''' + engine.execute(sql) + +class PluginInfo(CkanCommand): + ''' Provide info on installed plugins. + ''' + + summary = __doc__.split('\n')[0] + usage = __doc__ + max_args = 0 + min_args = 0 + + def command(self): + self.get_info() + + def get_info(self): + ''' print info about current plugins from the .ini file''' + import ckan.plugins as p + self._load_config() + interfaces = {} + plugins = {} + for name in dir(p): + item = getattr(p, name) + try: + if issubclass(item, p.Interface): + interfaces[item] = {'class' : item} + except TypeError: + pass + + for interface in interfaces: + for plugin in p.PluginImplementations(interface): + name = plugin.name + if name not in plugins: + plugins[name] = {'doc' : plugin.__doc__, + 'class' : plugin, + 'implements' : []} + plugins[name]['implements'].append(interface.__name__) + + for plugin in plugins: + p = plugins[plugin] + print plugin + ':' + print '-' * (len(plugin) + 1) + if p['doc']: + print p['doc'] + print 'Implements:' + for i in p['implements']: + extra = None + if i == 'ITemplateHelpers': + extra = self.template_helpers(p['class']) + if i == 'IActions': + extra = self.actions(p['class']) + print ' %s' % i + if extra: + print extra + print + + + def actions(self, cls): + ''' Return readable action function info. ''' + actions = cls.get_actions() + return self.function_info(actions) + + def template_helpers(self, cls): + ''' Return readable helper function info. ''' + helpers = cls.get_helpers() + return self.function_info(helpers) + + def function_info(self, functions): + ''' Take a dict of functions and output readable info ''' + import inspect + output = [] + for function_name in functions: + fn = functions[function_name] + args_info = inspect.getargspec(fn) + params = args_info.args + num_params = len(params) + if args_info.varargs: + params.append('*' + args_info.varargs) + if args_info.keywords: + params.append('**' + args_info.keywords) + if args_info.defaults: + offset = num_params - len(args_info.defaults) + for i, v in enumerate(args_info.defaults): + params[i + offset] = params[i + offset] + '=' + repr(v) + # is this a classmethod if so remove the first parameter + if inspect.ismethod(fn) and inspect.isclass(fn.__self__): + params = params[1:] + params = ', '.join(params) + output.append(' %s(%s)' % (function_name, params)) + # doc string + if fn.__doc__: + bits = fn.__doc__.split('\n') + for bit in bits: + output.append(' %s' % bit) + return ('\n').join(output) diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py index 18e81beee4f..44d5c646f21 100644 --- a/ckan/lib/create_test_data.py +++ b/ckan/lib/create_test_data.py @@ -6,13 +6,16 @@ class CreateTestData(cli.CkanCommand): '''Create test data in the database. Tests can also delete the created objects easily with the delete() method. - create-test-data - annakarenina and warandpeace - create-test-data search - realistic data to test search - create-test-data gov - government style data - create-test-data family - package relationships data - create-test-data user - create a user 'tester' with api key 'tester' + create-test-data - annakarenina and warandpeace + create-test-data search - realistic data to test search + create-test-data gov - government style data + create-test-data family - package relationships data + create-test-data user - create a user 'tester' with api key 'tester' + create-test-data translations - annakarenina, warandpeace, and some test + translations of terms create-test-data vocabs - annakerenina, warandpeace, and some test vocabularies + ''' summary = __doc__.split('\n')[0] usage = __doc__ @@ -53,6 +56,8 @@ def command(self): self.create_gov_test_data() elif cmd == 'family': self.create_family_test_data() + elif cmd == 'translations': + self.create_translations_test_data() elif cmd == 'vocabs': self.create_vocabs_test_data() else: @@ -92,6 +97,44 @@ def create_test_user(cls): cls.user_refs.append(u'tester') @classmethod + + def create_translations_test_data(cls): + import ckan.model + CreateTestData.create() + rev = ckan.model.repo.new_revision() + rev.author = CreateTestData.author + rev.message = u'Creating test translations.' + + sysadmin_user = ckan.model.User.get('testsysadmin') + package = ckan.model.Package.get('annakarenina') + + # Add some new tags to the package. + # These tags are codes that are meant to be always translated before + # display, if not into the user's current language then into the + # fallback language. + package.add_tags([ckan.model.Tag('123'), ckan.model.Tag('456'), + ckan.model.Tag('789')]) + + # Add the above translations to CKAN. + for (lang_code, translations) in (('de', german_translations), + ('fr', french_translations), ('en', english_translations)): + for term in terms: + if term in translations: + data_dict = { + 'term': term, + 'term_translation': translations[term], + 'lang_code': lang_code, + } + context = { + 'model': ckan.model, + 'session': ckan.model.Session, + 'user': sysadmin_user.name, + } + ckan.logic.action.update.term_translation_update(context, + data_dict) + + ckan.model.Session.commit() + def create_vocabs_test_data(cls): import ckan.model CreateTestData.create() @@ -770,3 +813,49 @@ def get_all_data(cls): } } ] + +# Some test terms and translations. +terms = ('A Novel By Tolstoy', + 'Index of the novel', + 'russian', + 'tolstoy', + "Dave's books", + "Roger's books", + 'Other (Open)', + 'romantic novel', + 'book', + '123', + '456', + '789', + 'plain text', + 'Roger likes these books.', +) +english_translations = { + '123': 'jealousy', + '456': 'realism', + '789': 'hypocrisy', +} +german_translations = { + 'A Novel By Tolstoy': 'Roman von Tolstoi', + 'Index of the novel': 'Index des Romans', + 'russian': 'Russisch', + 'tolstoy': 'Tolstoi', + "Dave's books": 'Daves Bucher', + "Roger's books": 'Rogers Bucher', + 'Other (Open)': 'Andere (Open)', + 'romantic novel': 'Liebesroman', + 'book': 'Buch', + '456': 'Realismus', + '789': 'Heuchelei', + 'plain text': 'Klartext', + 'Roger likes these books.': 'Roger mag diese Bucher.' +} +french_translations = { + 'A Novel By Tolstoy': 'A Novel par Tolstoi', + 'Index of the novel': 'Indice du roman', + 'russian': 'russe', + 'romantic novel': 'roman romantique', + 'book': 'livre', + '123': 'jalousie', + '789': 'hypocrisie', +} diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index d0e7d6c11e5..75fe85ec14d 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -1,7 +1,7 @@ import datetime from pylons import config from sqlalchemy.sql import select - +import datetime import ckan.model import ckan.misc import ckan.logic as logic @@ -32,6 +32,11 @@ def group_list_dictize(obj_list, context, group_dict['packages'] = len(obj.active_packages().all()) + if context.get('for_view'): + for item in plugins.PluginImplementations( + plugins.IGroupController): + group_dict = item.before_view(group_dict) + result_list.append(group_dict) return sorted(result_list, key=sort_key, reverse=reverse) @@ -43,10 +48,20 @@ def resource_list_dictize(res_list, context): resource_dict = resource_dictize(res, context) if active and res.state not in ('active', 'pending'): continue + result_list.append(resource_dict) return sorted(result_list, key=lambda x: x["position"]) +def related_list_dictize(related_list, context): + result_list = [] + for res in related_list: + related_dict = related_dictize(res, context) + result_list.append(related_dict) + + return sorted(result_list, key=lambda x: x["created"], reverse=True) + + def extras_dict_dictize(extras_dict, context): result_list = [] for name, extra in extras_dict.iteritems(): @@ -81,8 +96,15 @@ def resource_dictize(res, context): extras = resource.pop("extras", None) if extras: resource.update(extras) + #tracking + model = context['model'] + tracking = model.TrackingSummary.get_for_resource(res.url) + resource['tracking_summary'] = tracking return resource +def related_dictize(rel, context): + return d.table_dictize(rel, context) + def _execute_with_revision(q, rev_table, context): ''' Takes an SqlAlchemy query (q) that is (at its base) a Select on an @@ -153,6 +175,7 @@ def package_dictize(pkg, context): q = q.where(resource_group.c.package_id == pkg.id) result = _execute_with_revision(q, res_rev, context) result_dict["resources"] = resource_list_dictize(result, context) + #tags tag_rev = model.package_tag_revision_table tag = model.tag_table @@ -161,11 +184,22 @@ def package_dictize(pkg, context): ).where(tag_rev.c.package_id == pkg.id) result = _execute_with_revision(q, tag_rev, context) result_dict["tags"] = d.obj_list_dictize(result, context, lambda x: x["name"]) + + # Add display_names to tags. At first a tag's display_name is just the + # same as its name, but the display_name might get changed later (e.g. + # translated into another language by the multilingual extension). + for tag in result_dict['tags']: + assert not tag.has_key('display_name') + tag['display_name'] = tag['name'] + #extras extra_rev = model.extra_revision_table q = select([extra_rev]).where(extra_rev.c.package_id == pkg.id) result = _execute_with_revision(q, extra_rev, context) result_dict["extras"] = extras_list_dictize(result, context) + #tracking + tracking = model.TrackingSummary.get_for_package(pkg.id) + result_dict['tracking_summary'] = tracking #groups member_rev = model.member_revision_table group = model.group_table @@ -210,10 +244,9 @@ def package_dictize(pkg, context): if pkg.metadata_created else None if context.get('for_view'): - for item in plugins.PluginImplementations(plugins.IPackageController): + for item in plugins.PluginImplementations( plugins.IPackageController): result_dict = item.before_view(result_dict) - return result_dict def _get_members(context, group, member_type): @@ -226,7 +259,6 @@ def _get_members(context, group, member_type): filter(model.Member.state == 'active').\ filter(model.Member.table_name == member_type[:-1]).all() - def group_dictize(group, context): model = context['model'] result_dict = d.table_dictize(group, context) @@ -279,6 +311,18 @@ def tag_dictize(tag, context): result_dict = d.table_dictize(tag, context) result_dict["packages"] = d.obj_list_dictize(tag.packages, context) + + # Add display_names to tags. At first a tag's display_name is just the + # same as its name, but the display_name might get changed later (e.g. + # translated into another language by the multilingual extension). + assert not result_dict.has_key('display_name') + result_dict['display_name'] = result_dict['name'] + + if context.get('for_view'): + for item in plugins.PluginImplementations( + plugins.ITagController): + result_dict = item.before_view(result_dict) + return result_dict def user_list_dictize(obj_list, context, diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py index aa9f1a5f45e..8001dd4c49b 100644 --- a/ckan/lib/dictization/model_save.py +++ b/ckan/lib/dictization/model_save.py @@ -1,3 +1,4 @@ +import datetime import uuid from sqlalchemy.orm import class_mapper import ckan.lib.dictization as d @@ -8,7 +9,6 @@ def resource_dict_save(res_dict, context): model = context["model"] session = context["session"] - trigger_url_change = False id = res_dict.get("id") obj = None @@ -26,9 +26,12 @@ def resource_dict_save(res_dict, context): for key, value in res_dict.iteritems(): if isinstance(value, list): continue - if key in ('extras', 'revision_timestamp'): + if key in ('extras', 'revision_timestamp', 'tracking_summary'): continue if key in fields: + if isinstance(getattr(obj, key), datetime.datetime): + if getattr(obj, key).isoformat() == value: + continue if key == 'url' and not new and obj.url <> value: obj.url_changed = True setattr(obj, key, value) @@ -417,6 +420,14 @@ def user_dict_save(user_dict, context): return user + +def related_dict_save(related_dict, context): + model = context['model'] + session = context['session'] + + return d.table_dict_save(related_dict,model.Related, context) + + def package_api_to_dict(api1_dict, context): package = context.get("package") diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index ac1d053273a..78924b4ff24 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -7,6 +7,7 @@ """ import email.utils import datetime +import logging import re import urllib @@ -45,6 +46,8 @@ except ImportError: import simplejson as json +_log = logging.getLogger(__name__) + def redirect_to(*args, **kw): '''A routes.redirect_to wrapper to retain the i18n settings''' kw['__ckan_no_root'] = True @@ -109,6 +112,9 @@ def _add_i18n_to_url(url_to_amend, **kw): root = request.environ.get('SCRIPT_NAME', '') except TypeError: root = '' + if kw.get('qualified', False): + # if qualified is given we want the full url ie http://... + root = _routes_default_url_for('/', qualified=True)[:-1] + root # ckan.root_path is defined when we have none standard language # position in the url root_path = config.get('ckan.root_path', None) @@ -331,8 +337,41 @@ def _subnav_named_route(text, routename, **kwargs): def default_group_type(): return str( config.get('ckan.default.group_type', 'group') ) +def unselected_facet_items(facet, limit=10): + '''Return the list of unselected facet items for the given facet, sorted + by count. + + Returns the list of unselected facet contraints or facet items (e.g. tag + names like "russian" or "tolstoy") for the given search facet (e.g. + "tags"), sorted by facet item count (i.e. the number of search results that + match each facet item). + + Reads the complete list of facet items for the given facet from + c.search_facets, and filters out the facet items that the user has already + selected. + + Arguments: + facet -- the name of the facet to filter. + limit -- the max. number of facet items to return. + + ''' + if not c.search_facets or \ + not c.search_facets.get(facet) or \ + not c.search_facets.get(facet).get('items'): + return [] + facets = [] + for facet_item in c.search_facets.get(facet)['items']: + if not len(facet_item['name'].strip()): + continue + if not (facet, facet_item['name']) in request.params.items(): + facets.append(facet_item) + return sorted(facets, key=lambda item: item['count'], reverse=True)[:limit] def facet_items(*args, **kwargs): + """ + DEPRECATED: Use the new facet data structure, and `unselected_facet_items()` + """ + _log.warning('Deprecated function: ckan.lib.helpers:facet_items(). Will be removed in v1.8') # facet_items() used to need c passing as the first arg # this is depriciated as pointless # throws error if ckan.restrict_template_vars is True diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py index c7fe22875d9..258e1449681 100644 --- a/ckan/lib/plugins.py +++ b/ckan/lib/plugins.py @@ -274,6 +274,9 @@ def setup_template_variables(self, context, data_dict): c.licences = [('', '')] + base.model.Package.get_license_options() c.is_sysadmin = authz.Authorizer().is_sysadmin(c.user) + if c.pkg: + c.related_count = len(c.pkg.related) + ## This is messy as auths take domain object not data_dict context_pkg = context.get('package', None) pkg = context_pkg or c.pkg @@ -287,6 +290,7 @@ def setup_template_variables(self, context, data_dict): c.auth_for_change_state = False + class DefaultGroupForm(object): """ Provides a default implementation of the pluggable Group controller diff --git a/ckan/lib/search/index.py b/ckan/lib/search/index.py index 2599115a0ff..086a39ed79d 100644 --- a/ckan/lib/search/index.py +++ b/ckan/lib/search/index.py @@ -132,6 +132,12 @@ def index_package(self, pkg_dict): pkg_dict['groups'] = [group['name'] for group in groups] + # tracking + tracking_summary = pkg_dict.pop('tracking_summary', None) + if tracking_summary: + pkg_dict['views_total'] = tracking_summary['total'] + pkg_dict['views_recent'] = tracking_summary['recent'] + # flatten the structure for indexing: for resource in pkg_dict.get('resources', []): for (okey, nkey) in [('description', 'res_description'), diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index 513f2918ab3..30cd4ae7983 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -39,6 +39,7 @@ class NotAuthorized(ActionError): class ParameterError(ActionError): pass + class ValidationError(ParameterError): def __init__(self, error_dict, error_summary=None, extra_msg=None): self.error_dict = error_dict @@ -224,3 +225,29 @@ def get_action(action): _actions.update(fetched_actions) return _actions.get(action) +def get_or_bust(data_dict, keys): + '''Try and get values from dictionary and if they are not there + raise a validataion error. + + data_dict: a dictionary + keys: either a single string key in which case will return a single value, + or a iterable which will return a tuple for unpacking purposes. + + e.g single_value = get_or_bust(data_dict, 'a_key') + value_1, value_2 = get_or_bust(data_dict, ['key1', 'key2']) + ''' + values = [] + errors = {} + + if isinstance(keys, basestring): + keys = [keys] + for key in keys: + value = data_dict.get(key) + if not value: + errors[key] = _('Missing value') + values.append(value) + if errors: + raise ValidationError(errors) + if len(values) == 1: + return values[0] + return tuple(values) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index d142dcf3744..aee73a32aa4 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -112,6 +112,35 @@ def resource_create(context, data_dict): ckan.logic.schema.default_resource_schema(), context) + +def related_create(context, data_dict): + model = context['model'] + user = context['user'] + userobj = model.User.get(user) + + data_dict["owner_id"] = userobj.id + data, errors = validate(data_dict, + ckan.logic.schema.default_related_schema(), + context) + if errors: + model.Session.rollback() + raise ValidationError(errors, error_summary(errors)) + + related = model_save.related_dict_save(data, context) + if not context.get('defer_commit'): + model.repo.commit_and_remove() + + if 'dataset_id' in data_dict: + dataset = model.Package.get(data_dict['dataset_id']) + dataset.related.append( related ) + model.repo.commit_and_remove() + + context["related"] = related + context["id"] = related.id + log.debug('Created object %s' % str(related.title)) + return model_dictize.related_dictize(related, context) + + def package_relationship_create(context, data_dict): model = context['model'] diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index a8368ec6195..c806a5113cd 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -64,6 +64,21 @@ def package_relationship_delete(context, data_dict): relationship.delete() model.repo.commit() +def related_delete(context, data_dict): + model = context['model'] + user = context['user'] + id = data_dict['id'] + + entity = model.Related.get(id) + + if entity is None: + raise NotFound + + check_access('related_delete',context, data_dict) + + entity.delete() + model.repo.commit() + def member_delete(context, data_dict=None): """ diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 21429d357d0..fd0ce25ad57 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -6,7 +6,7 @@ import webhelpers.html from sqlalchemy.sql import select from sqlalchemy.orm import aliased -from sqlalchemy import or_, and_, func, desc, case +from sqlalchemy import or_, and_, func, desc, case, text import ckan import ckan.authz @@ -31,6 +31,7 @@ check_access = logic.check_access NotFound = logic.NotFound ValidationError = logic.ValidationError +get_or_bust = logic.get_or_bust def _package_list_with_resources(context, package_revision_list): package_list = [] @@ -105,6 +106,69 @@ def package_revision_list(context, data_dict): include_groups=False)) return revision_dicts + +def related_show(context, data_dict=None): + """ + Shows a single related item + + context: + model - The CKAN model module + user - The name of the current user + + data_dict: + id - The ID of the related item we want to show + """ + model = context['model'] + id = data_dict['id'] + + related = model.Related.get(id) + context['related'] = related + + if related is None: + raise NotFound + + check_access('related_show',context, data_dict) + + schema = context.get('schema') or ckan.logic.schema.default_related_schema() + related_dict = model_dictize.related_dictize(related, context) + related_dict, errors = validate(related_dict, schema, context=context) + + return related_dict + + +def related_list(context, data_dict=None): + """ + List the related items for a specific package which should be + mentioned in the data_dict + + context: + model - The CKAN model module + user - The name of the current user + session - The current DB session + + data_dict: + id - The ID of the dataset to which we want to list related items + or + dataset - The dataset (package) model + """ + model = context['model'] + session = context['session'] + dataset = data_dict.get('dataset', None) + + if not dataset: + dataset = model.Package.get(data_dict.get('id')) + + if not dataset: + raise NotFound + + check_access('related_show',context, data_dict) + + relateds = model.Related.get_for_dataset(dataset, status='active') + related_items = (r.related for r in relateds) + related_list = model_dictize.related_list_dictize( related_items, context) + return related_list + + def member_list(context, data_dict=None): """ Returns a list of (id,type,capacity) tuples that are members of the @@ -417,9 +481,27 @@ def resource_show(context, data_dict): raise NotFound check_access('resource_show', context, data_dict) - return model_dictize.resource_dictize(resource, context) +def resource_status_show(context, data_dict): + + model = context['model'] + id = get_or_bust(data_dict, 'id') + + check_access('resource_status_show', context, data_dict) + + # needs to be text query as celery tables are not in our model + q = text("""select status, date_done, traceback, task_status.* + from task_status left join celery_taskmeta + on task_status.value = celery_taskmeta.task_id and key = 'celery_task_id' + where entity_id = :entity_id """) + + result = model.Session.connection().execute(q, entity_id=id) + result_list = [table_dictize(row, context) for row in result] + + return result_list + + def revision_show(context, data_dict): model = context['model'] api = context.get('api_version') @@ -753,12 +835,40 @@ def package_search(context, data_dict): 'results': results } + # Transform facets into a more useful data structure. + restructured_facets = {} + for key, value in search_results['facets'].items(): + restructured_facets[key] = { + 'title': key, + 'items': [] + } + for key_, value_ in value.items(): + new_facet_dict = {} + new_facet_dict['name'] = key_ + if key == 'groups': + group = model.Group.get(key_) + if group: + new_facet_dict['display_name'] = group.display_name + else: + new_facet_dict['display_name'] = key_ + else: + new_facet_dict['display_name'] = key_ + new_facet_dict['count'] = value_ + restructured_facets[key]['items'].append(new_facet_dict) + search_results['search_facets'] = restructured_facets + # check if some extension needs to modify the search results for item in plugins.PluginImplementations(plugins.IPackageController): search_results = item.after_search(search_results,data_dict) - return search_results + # After extensions have had a chance to modify the facets, sort them by + # display name. + for facet in search_results['search_facets']: + search_results['search_facets'][facet]['items'] = sorted( + search_results['search_facets'][facet]['items'], + key=lambda facet: facet['display_name'], reverse=True) + return search_results def resource_search(context, data_dict): model = context['model'] @@ -933,13 +1043,13 @@ def term_translation_show(context, data_dict): q = select([trans_table]) - if 'term' not in data_dict: - raise ValidationError({'term': 'term not in data'}) + if 'terms' not in data_dict: + raise ValidationError({'terms': 'terms not in data'}) - q = q.where(trans_table.c.term == data_dict['term']) + q = q.where(trans_table.c.term.in_(data_dict['terms'])) - if 'lang_code' in data_dict: - q = q.where(trans_table.c.lang_code == data_dict['lang_code']) + if 'lang_codes' in data_dict: + q = q.where(trans_table.c.lang_code.in_(data_dict['lang_codes'])) conn = model.Session.connection() cursor = conn.execute(q) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 2509c924af6..cd5aa2fe8da 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -98,6 +98,35 @@ def make_latest_pending_package_active(context, data_dict): session.remove() +def related_update(context, data_dict): + model = context['model'] + user = context['user'] + id = data_dict["id"] + + schema = context.get('schema') or ckan.logic.schema.default_related_schema() + model.Session.remove() + + related = model.Related.get(id) + context["related"] = related + + if not related: + logging.error('Could not find related ' + id) + raise NotFound(_('Related was not found.')) + + check_access('related_update', context, data_dict) + data, errors = validate(data_dict, schema, context) + + if errors: + model.Session.rollback() + raise ValidationError(errors, error_summary(errors)) + + related = model_save.related_dict_save(data, context) + if not context.get('defer_commit'): + model.repo.commit() + return model_dictize.related_dictize(related, context) + + + def resource_update(context, data_dict): model = context['model'] user = context['user'] diff --git a/ckan/logic/auth/__init__.py b/ckan/logic/auth/__init__.py index ab3ee7aa222..7b00f9c8786 100644 --- a/ckan/logic/auth/__init__.py +++ b/ckan/logic/auth/__init__.py @@ -17,6 +17,9 @@ def _get_object(context, data_dict, name, class_name): obj = context[name] return obj +def get_related_object(context, data_dict = {}): + return _get_object(context, data_dict, 'related', 'Related') + def get_package_object(context, data_dict = {}): return _get_object(context, data_dict, 'package', 'Package') diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index 3ec56088ddb..546a1b872ac 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -1,12 +1,11 @@ -from ckan.logic import check_access_old, NotFound +import ckan.logic as logic from ckan.authz import Authorizer from ckan.lib.base import _ - def package_create(context, data_dict=None): model = context['model'] user = context['user'] - check1 = check_access_old(model.System(), model.Action.PACKAGE_CREATE, context) + check1 = logic.check_access_old(model.System(), model.Action.PACKAGE_CREATE, context) if not check1: return {'success': False, 'msg': _('User %s not authorized to create packages') % str(user)} @@ -18,6 +17,17 @@ def package_create(context, data_dict=None): return {'success': True} +def related_create(context, data_dict=None): + model = context['model'] + user = context['user'] + userobj = model.User.get( user ) + + if userobj: + return {'success': True} + + return {'success': False, 'msg': _('You must be logged in to add a related item')} + + def resource_create(context, data_dict): return {'success': False, 'msg': 'Not implemented yet in the auth refactor'} @@ -43,7 +53,7 @@ def group_create(context, data_dict=None): model = context['model'] user = context['user'] - authorized = check_access_old(model.System(), model.Action.GROUP_CREATE, context) + authorized = logic.check_access_old(model.System(), model.Action.GROUP_CREATE, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to create groups') % str(user)} else: @@ -53,7 +63,7 @@ def authorization_group_create(context, data_dict=None): model = context['model'] user = context['user'] - authorized = check_access_old(model.System(), model.Action.AUTHZ_GROUP_CREATE, context) + authorized = logic.check_access_old(model.System(), model.Action.AUTHZ_GROUP_CREATE, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to create authorization groups') % str(user)} else: @@ -67,7 +77,7 @@ def user_create(context, data_dict=None): model = context['model'] user = context['user'] - authorized = check_access_old(model.System(), model.Action.USER_CREATE, context) + authorized = logic.check_access_old(model.System(), model.Action.USER_CREATE, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to create users') % str(user)} else: @@ -98,7 +108,7 @@ def _check_group_auth(context, data_dict): id = group_blob grp = model.Group.get(id) if grp is None: - raise NotFound(_('Group was not found.')) + raise logic.NotFound(_('Group was not found.')) groups.add(grp) if pkg: @@ -107,7 +117,7 @@ def _check_group_auth(context, data_dict): groups = groups - set(pkg_groups) for group in groups: - if not check_access_old(group, model.Action.EDIT, context): + if not logic.check_access_old(group, model.Action.EDIT, context): return False return True diff --git a/ckan/logic/auth/delete.py b/ckan/logic/auth/delete.py index 12937c284d9..c99bec37bc5 100644 --- a/ckan/logic/auth/delete.py +++ b/ckan/logic/auth/delete.py @@ -1,5 +1,5 @@ -from ckan.logic import check_access_old -from ckan.logic.auth import get_package_object, get_group_object +import ckan.logic as logic +from ckan.logic.auth import get_package_object, get_group_object, get_related_object from ckan.logic.auth.create import package_relationship_create from ckan.authz import Authorizer from ckan.lib.base import _ @@ -9,22 +9,40 @@ def package_delete(context, data_dict): user = context['user'] package = get_package_object(context, data_dict) - authorized = check_access_old(package, model.Action.PURGE, context) + authorized = logic.check_access_old(package, model.Action.PURGE, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to delete package %s') % (str(user),package.id)} else: return {'success': True} + +def related_delete(context, data_dict): + model = context['model'] + user = context['user'] + if not user: + return {'success': False, 'msg': _('Only the owner can delete a related item')} + + if Authorizer().is_sysadmin(unicode(user)): + return {'success': True} + + related = get_related_object(context, data_dict) + userobj = model.User.get( user ) + if not userobj or userobj.id != related.owner_id: + return {'success': False, 'msg': _('Only the owner can delete a related item')} + + return {'success': True} + + def package_relationship_delete(context, data_dict): can_edit_this_relationship = package_relationship_create(context, data_dict) if not can_edit_this_relationship['success']: return can_edit_this_relationship - + model = context['model'] user = context['user'] relationship = context['relationship'] - authorized = check_access_old(relationship, model.Action.PURGE, context) + authorized = logic.check_access_old(relationship, model.Action.PURGE, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to delete relationship %s') % (str(user),relationship.id)} else: @@ -35,7 +53,7 @@ def group_delete(context, data_dict): user = context['user'] group = get_group_object(context, data_dict) - authorized = check_access_old(group, model.Action.PURGE, context) + authorized = logic.check_access_old(group, model.Action.PURGE, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to delete group %s') % (str(user),group.id)} else: diff --git a/ckan/logic/auth/get.py b/ckan/logic/auth/get.py index 27e8436dfe5..2c32746ae09 100644 --- a/ckan/logic/auth/get.py +++ b/ckan/logic/auth/get.py @@ -1,7 +1,8 @@ -from ckan.logic import check_access_old, NotFound +import ckan.logic as logic from ckan.authz import Authorizer from ckan.lib.base import _ -from ckan.logic.auth import get_package_object, get_group_object, get_resource_object +from ckan.logic.auth import (get_package_object, get_group_object, + get_resource_object, get_related_object) def site_read(context, data_dict): @@ -84,12 +85,16 @@ def package_show(context, data_dict): user = context.get('user') package = get_package_object(context, data_dict) - authorized = check_access_old(package, model.Action.READ, context) + authorized = logic.check_access_old(package, model.Action.READ, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to read package %s') % (str(user),package.id)} else: return {'success': True} +def related_show(context, data_dict=None): + return {'success': True} + + def resource_show(context, data_dict): model = context['model'] user = context.get('user') @@ -102,11 +107,11 @@ def resource_show(context, data_dict): .filter(model.ResourceGroup.id == resource.resource_group_id) pkg = query.first() if not pkg: - raise NotFound(_('No package found for this resource, cannot check auth.')) - + raise logic.NotFound(_('No package found for this resource, cannot check auth.')) + pkg_dict = {'id': pkg.id} authorized = package_show(context, pkg_dict).get('success') - + if not authorized: return {'success': False, 'msg': _('User %s not authorized to read resource %s') % (str(user), resource.id)} else: @@ -121,7 +126,7 @@ def group_show(context, data_dict): user = context.get('user') group = get_group_object(context, data_dict) - authorized = check_access_old(group, model.Action.READ, context) + authorized = logic.check_access_old(group, model.Action.READ, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to read group %s') % (str(user),group.id)} else: @@ -154,6 +159,9 @@ def format_autocomplete(context, data_dict): def task_status_show(context, data_dict): return {'success': True} +def resource_status_show(context, data_dict): + return {'success': True} + ## Modifications for rest api def package_show_rest(context, data_dict): diff --git a/ckan/logic/auth/publisher/create.py b/ckan/logic/auth/publisher/create.py index 17eb77d38d6..d1bece7ebaf 100644 --- a/ckan/logic/auth/publisher/create.py +++ b/ckan/logic/auth/publisher/create.py @@ -1,7 +1,7 @@ -from ckan.logic.auth import get_package_object, get_group_object, \ - get_user_object, get_resource_object +from ckan.logic.auth import (get_package_object, get_group_object, + get_user_object, get_resource_object, get_related_object) from ckan.logic.auth.publisher import _groups_intersect -from ckan.logic import NotFound +import ckan.logic as logic from ckan.authz import Authorizer from ckan.lib.base import _ @@ -19,6 +19,18 @@ def package_create(context, data_dict=None): return {'success': False, 'msg': 'You must be logged in to create a package'} + +def related_create(context, data_dict=None): + model = context['model'] + user = context['user'] + userobj = model.User.get( user ) + + if userobj: + return {'success': True} + + return {'success': False, 'msg': _('You must be logged in to add a related item')} + + def resource_create(context, data_dict): return {'success': False, 'msg': 'Not implemented yet in the auth refactor'} @@ -68,7 +80,7 @@ def group_create(context, data_dict=None): # If the user is doing this within another group then we need to make sure that # the user has permissions for this group. group = get_group_object( context ) - except NotFound: + except logic.NotFound: return { 'success' : True } userobj = model.User.get( user ) diff --git a/ckan/logic/auth/publisher/delete.py b/ckan/logic/auth/publisher/delete.py index 00d26afdcbc..9d3388f2340 100644 --- a/ckan/logic/auth/publisher/delete.py +++ b/ckan/logic/auth/publisher/delete.py @@ -1,6 +1,6 @@ +import ckan.logic as logic from ckan.logic.auth import get_package_object, get_group_object, \ - get_user_object, get_resource_object -from ckan.logic.auth import get_package_object, get_group_object + get_user_object, get_resource_object, get_related_object from ckan.logic.auth.publisher import _groups_intersect from ckan.logic.auth.publisher.create import package_relationship_create from ckan.authz import Authorizer @@ -29,6 +29,23 @@ def package_delete(context, data_dict): def package_relationship_delete(context, data_dict): return package_relationship_create(context, data_dict) +def related_delete(context, data_dict): + model = context['model'] + user = context['user'] + if not user: + return {'success': False, 'msg': _('Only the owner can delete a related item')} + + if Authorizer().is_sysadmin(unicode(user)): + return {'success': True} + + related = get_related_object(context, data_dict) + userobj = model.User.get( user ) + if not userobj or userobj.id != related.owner_id: + return {'success': False, 'msg': _('Only the owner can delete a related item')} + + return {'success': True} + + def group_delete(context, data_dict): """ Group delete permission. Checks that the user specified is within the group to be deleted diff --git a/ckan/logic/auth/publisher/get.py b/ckan/logic/auth/publisher/get.py index b2dbd9a9d86..3a76048817d 100644 --- a/ckan/logic/auth/publisher/get.py +++ b/ckan/logic/auth/publisher/get.py @@ -1,5 +1,6 @@ +import ckan.logic as logic from ckan.logic.auth import get_package_object, get_group_object, \ - get_user_object, get_resource_object + get_user_object, get_resource_object, get_related_object from ckan.lib.base import _ from ckan.logic.auth.publisher import _groups_intersect from ckan.authz import Authorizer @@ -97,6 +98,10 @@ def package_show(context, data_dict): return {'success': True} +def related_show(context, data_dict=None): + return {'success': True} + + def resource_show(context, data_dict): """ Resource show permission checks the user group if the package state is deleted """ model = context['model'] @@ -160,6 +165,9 @@ def format_autocomplete(context, data_dict): def task_status_show(context, data_dict): return {'success': True} +def resource_status_show(context, data_dict): + return {'success': True} + ## Modifications for rest api def package_show_rest(context, data_dict): diff --git a/ckan/logic/auth/publisher/update.py b/ckan/logic/auth/publisher/update.py index dbbf1942ebd..30d7a1d92bd 100644 --- a/ckan/logic/auth/publisher/update.py +++ b/ckan/logic/auth/publisher/update.py @@ -1,5 +1,7 @@ +import ckan.logic as logic from ckan.logic.auth import get_package_object, get_group_object, \ - get_user_object, get_resource_object + get_user_object, get_resource_object, get_related_object, \ + get_authorization_group_object from ckan.logic.auth.publisher import _groups_intersect from ckan.logic.auth.publisher.create import package_relationship_create from ckan.authz import Authorizer @@ -86,6 +88,19 @@ def group_update(context, data_dict): return { 'success': True } +def related_update(context, data_dict): + model = context['model'] + user = context['user'] + if not user: + return {'success': False, 'msg': _('Only the owner can update a related item')} + + related = get_related_object(context, data_dict) + userobj = model.User.get( user ) + if not userobj or userobj.id != related.owner_id: + return {'success': False, 'msg': _('Only the owner can update a related item')} + + return {'success': True} + def group_change_state(context, data_dict): return group_update(context, data_dict) diff --git a/ckan/logic/auth/update.py b/ckan/logic/auth/update.py index 93036b75b04..afd00688272 100644 --- a/ckan/logic/auth/update.py +++ b/ckan/logic/auth/update.py @@ -1,6 +1,7 @@ -from ckan.logic import check_access_old, NotFound -from ckan.logic.auth import get_package_object, get_resource_object, get_group_object, get_authorization_group_object, \ - get_user_object, get_resource_object +import ckan.logic as logic +from ckan.logic.auth import (get_package_object, get_resource_object, + get_group_object, get_authorization_group_object, + get_user_object, get_resource_object, get_related_object) from ckan.logic.auth.create import _check_group_auth, package_relationship_create from ckan.authz import Authorizer from ckan.lib.base import _ @@ -13,7 +14,7 @@ def package_update(context, data_dict): user = context.get('user') package = get_package_object(context, data_dict) - check1 = check_access_old(package, model.Action.EDIT, context) + check1 = logic.check_access_old(package, model.Action.EDIT, context) if not check1: return {'success': False, 'msg': _('User %s not authorized to edit package %s') % (str(user), package.id)} else: @@ -35,11 +36,11 @@ def resource_update(context, data_dict): .filter(model.ResourceGroup.id == resource.resource_group_id) pkg = query.first() if not pkg: - raise NotFound(_('No package found for this resource, cannot check auth.')) - + raise logic.NotFound(_('No package found for this resource, cannot check auth.')) + pkg_dict = {'id': pkg.id} authorized = package_update(context, pkg_dict).get('success') - + if not authorized: return {'success': False, 'msg': _('User %s not authorized to read edit %s') % (str(user), resource.id)} else: @@ -53,7 +54,7 @@ def package_change_state(context, data_dict): user = context['user'] package = get_package_object(context, data_dict) - authorized = check_access_old(package, model.Action.CHANGE_STATE, context) + authorized = logic.check_access_old(package, model.Action.CHANGE_STATE, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to change state of package %s') % (str(user),package.id)} else: @@ -64,7 +65,7 @@ def package_edit_permissions(context, data_dict): user = context['user'] package = get_package_object(context, data_dict) - authorized = check_access_old(package, model.Action.EDIT_PERMISSIONS, context) + authorized = logic.check_access_old(package, model.Action.EDIT_PERMISSIONS, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to edit permissions of package %s') % (str(user),package.id)} else: @@ -74,19 +75,33 @@ def group_update(context, data_dict): model = context['model'] user = context['user'] group = get_group_object(context, data_dict) - - authorized = check_access_old(group, model.Action.EDIT, context) + + authorized = logic.check_access_old(group, model.Action.EDIT, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to edit group %s') % (str(user),group.id)} else: return {'success': True} +def related_update(context, data_dict): + model = context['model'] + user = context['user'] + if not user: + return {'success': False, 'msg': _('Only the owner can update a related item')} + + related = get_related_object(context, data_dict) + userobj = model.User.get( user ) + if not userobj or userobj.id != related.owner_id: + return {'success': False, 'msg': _('Only the owner can update a related item')} + + return {'success': True} + + def group_change_state(context, data_dict): model = context['model'] user = context['user'] group = get_group_object(context, data_dict) - authorized = check_access_old(group, model.Action.CHANGE_STATE, context) + authorized = logic.check_access_old(group, model.Action.CHANGE_STATE, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to change state of group %s') % (str(user),group.id)} else: @@ -97,7 +112,7 @@ def group_edit_permissions(context, data_dict): user = context['user'] group = get_group_object(context, data_dict) - authorized = check_access_old(group, model.Action.EDIT_PERMISSIONS, context) + authorized = logic.check_access_old(group, model.Action.EDIT_PERMISSIONS, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to edit permissions of group %s') % (str(user),group.id)} else: @@ -108,7 +123,7 @@ def authorization_group_update(context, data_dict): user = context['user'] authorization_group = get_authorization_group_object(context, data_dict) - authorized = check_access_old(authorization_group, model.Action.EDIT, context) + authorized = logic.check_access_old(authorization_group, model.Action.EDIT, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to edit permissions of authorization group %s') % (str(user),authorization_group.id)} else: @@ -119,7 +134,7 @@ def authorization_group_edit_permissions(context, data_dict): user = context['user'] authorization_group = get_authorization_group_object(context, data_dict) - authorized = check_access_old(authorization_group, model.Action.EDIT_PERMISSIONS, context) + authorized = logic.check_access_old(authorization_group, model.Action.EDIT_PERMISSIONS, context) if not authorized: return {'success': False, 'msg': _('User %s not authorized to edit permissions of authorization group %s') % (str(user),authorization_group.id)} else: @@ -152,7 +167,7 @@ def task_status_update(context, data_dict): if 'ignore_auth' in context and context['ignore_auth']: return {'success': True} - + authorized = Authorizer().is_sysadmin(unicode(user)) if not authorized: return {'success': False, 'msg': _('User %s not authorized to update task_status table') % str(user)} diff --git a/ckan/logic/converters.py b/ckan/logic/converters.py index 5de75a7f1e4..6b927ae3106 100644 --- a/ckan/logic/converters.py +++ b/ckan/logic/converters.py @@ -60,13 +60,11 @@ def callable(key, data, errors, context): context['vocabulary'] = v for tag in new_tags: - tag_length_validator(tag, context) - tag_name_validator(tag, context) tag_in_vocabulary_validator(tag, context) for num, tag in enumerate(new_tags): - data[('tags', num+n, 'name')] = tag - data[('tags', num+n, 'vocabulary_id')] = v.id + data[('tags', num + n, 'name')] = tag + data[('tags', num + n, 'vocabulary_id')] = v.id return callable def convert_from_tags(vocab): diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index b07d2bb2d02..09ffe2217ae 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -62,9 +62,11 @@ def default_resource_schema(): 'webstore_url': [ignore_missing, unicode], 'cache_url': [ignore_missing, unicode], 'size': [ignore_missing, int_validator], + 'created': [ignore_missing, isodate], 'last_modified': [ignore_missing, isodate], 'cache_last_updated': [ignore_missing, isodate], 'webstore_last_updated': [ignore_missing, isodate], + 'tracking_summary': [ignore], '__extras': [ignore_missing, extras_unicode_convert, keep_extras], } @@ -85,6 +87,7 @@ def default_tags_schema(): 'vocabulary_id': [ignore_missing, unicode, vocabulary_id_exists], 'revision_timestamp': [ignore], 'state': [ignore], + 'display_name': [ignore], } return schema @@ -232,6 +235,20 @@ def default_update_group_schema(): return schema +def default_related_schema(): + schema = { + 'id': [ignore_missing, unicode], + 'title': [not_empty, unicode], + 'description': [ignore_missing, unicode], + 'type': [not_empty, unicode], + 'image_url': [ignore_missing, unicode], + 'url': [ignore_missing, unicode], + 'owner_id': [not_empty, unicode], + 'created': [ignore], + } + return schema + + def default_extras_schema(): schema = { diff --git a/ckan/migration/versions/054_add_resource_created_date.py b/ckan/migration/versions/054_add_resource_created_date.py new file mode 100644 index 00000000000..03641509c3e --- /dev/null +++ b/ckan/migration/versions/054_add_resource_created_date.py @@ -0,0 +1,9 @@ +def upgrade(migrate_engine): + migrate_engine.execute(''' + ALTER TABLE resource + ADD COLUMN created timestamp without time zone; + + ALTER TABLE resource_revision + ADD COLUMN created timestamp without time zone; + ''' + ) diff --git a/ckan/migration/versions/055_update_user_and_activity_detail.py b/ckan/migration/versions/055_update_user_and_activity_detail.py new file mode 100644 index 00000000000..5d349000333 --- /dev/null +++ b/ckan/migration/versions/055_update_user_and_activity_detail.py @@ -0,0 +1,9 @@ +def upgrade(migrate_engine): + migrate_engine.execute(''' + ALTER TABLE activity_detail + ALTER COLUMN activity_id DROP NOT NULL; + + ALTER TABLE "user" + ALTER COLUMN name SET NOT NULL; + ''' + ) diff --git a/ckan/migration/versions/056_add_related_table.py b/ckan/migration/versions/056_add_related_table.py new file mode 100644 index 00000000000..bcd909c7619 --- /dev/null +++ b/ckan/migration/versions/056_add_related_table.py @@ -0,0 +1,40 @@ +from sqlalchemy import * +from migrate import * + +def upgrade(migrate_engine): + metadata = MetaData() + metadata.bind = migrate_engine + migrate_engine.execute(''' +BEGIN; +CREATE TABLE related ( + id text NOT NULL, + type text NOT NULL, + title text, + description text, + image_url text, + url text, + created timestamp without time zone, + owner_id text +); + +CREATE TABLE related_dataset ( + id text NOT NULL, + dataset_id text NOT NULL, + related_id text NOT NULL, + status text +); + +ALTER TABLE related + ADD CONSTRAINT related_pkey PRIMARY KEY (id); + +ALTER TABLE related_dataset + ADD CONSTRAINT related_dataset_pkey PRIMARY KEY (id); + +ALTER TABLE related_dataset + ADD CONSTRAINT related_dataset_dataset_id_fkey FOREIGN KEY (dataset_id) REFERENCES package(id); + +ALTER TABLE related_dataset + ADD CONSTRAINT related_dataset_related_id_fkey FOREIGN KEY (related_id) REFERENCES related(id); +COMMIT; + ''' + ) diff --git a/ckan/migration/versions/057_tracking.py b/ckan/migration/versions/057_tracking.py new file mode 100644 index 00000000000..5f2fe43ec53 --- /dev/null +++ b/ckan/migration/versions/057_tracking.py @@ -0,0 +1,33 @@ +from sqlalchemy import * +from migrate import * + +def upgrade(migrate_engine): + migrate_engine.execute(''' + BEGIN; + CREATE TABLE tracking_raw ( + user_key character varying(100) NOT NULL, + url text NOT NULL, + tracking_type character varying(10) NOT NULL, + access_timestamp timestamp without time zone DEFAULT current_timestamp + ); + CREATE INDEX tracking_raw_url ON tracking_raw(url); + CREATE INDEX tracking_raw_user_key ON tracking_raw(user_key); + CREATE INDEX tracking_raw_access_timestamp ON tracking_raw(access_timestamp); + + CREATE TABLE tracking_summary( + url text NOT NULL, + package_id text, + tracking_type character varying(10) NOT NULL, + count int NOT NULL, + running_total int NOT NULL DEFAULT 0, + recent_views int NOT NULL DEFAULT 0, + tracking_date date + ); + + CREATE INDEX tracking_summary_url ON tracking_summary(url); + CREATE INDEX tracking_summary_package_id ON tracking_summary(package_id); + CREATE INDEX tracking_summary_date ON tracking_summary(tracking_date); + + COMMIT; + ''' + ) diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py index 8f891cdc5db..a3431fcdb10 100644 --- a/ckan/model/__init__.py +++ b/ckan/model/__init__.py @@ -19,12 +19,14 @@ import authz import package_extra import resource +import tracking import rating import package_relationship import task_status import vocabulary import activity import term_translation + import ckan.migration import ckan.lib.helpers as h @@ -128,6 +130,8 @@ term_translation_table = term_translation.term_translation_table +tracking_summary_table = tracking.tracking_summary_table +TrackingSummary = tracking.TrackingSummary # set up in init_model after metadata is bound version_table = None @@ -146,14 +150,14 @@ def init_model(engine): except sqlalchemy.exc.NoSuchTableError: pass - + class Repository(vdm.sqlalchemy.Repository): migrate_repository = ckan.migration.__path__[0] # note: tables_created value is not sustained between instantiations so # only useful for tests. The alternative is to use are_tables_created(). - tables_created_and_initialised = False + tables_created_and_initialised = False def init_db(self): '''Ensures tables, const data and some default config is created. @@ -174,6 +178,17 @@ def init_db(self): else: if not self.tables_created_and_initialised: self.upgrade_db() + ## make sure celery tables are made as celery only makes them after + ## adding a task + import ckan.lib.celery_app as celery_app + import celery.db.session as celery_session + + ##This creates the database tables it is a slight hack to celery. + backend = celery_app.celery.backend + celery_result_session = backend.ResultSession() + engine = celery_result_session.bind + celery_session.ResultModelBase.metadata.create_all(engine) + self.init_configuration_data() self.tables_created_and_initialised = True @@ -206,15 +221,20 @@ def init_configuration_data(self): rev = Revision() rev.author = 'system' rev.message = u'Initialising the Repository' +<<<<<<< HEAD meta.Session.add(rev) self.commit_and_remove() +======= + Session.add(rev) + self.commit_and_remove() +>>>>>>> master def create_db(self): '''Ensures tables, const data and some default config is created. i.e. the same as init_db APART from when running tests, when init_db has shortcuts. ''' - self.metadata.create_all(bind=self.metadata.bind) + self.metadata.create_all(bind=self.metadata.bind) self.init_const_data() self.init_configuration_data() @@ -238,7 +258,7 @@ def rebuild_db(self): self.session.remove() self.init_db() self.session.flush() - + def delete_all(self): '''Delete all data from all tables.''' self.session.remove() @@ -275,7 +295,7 @@ def upgrade_db(self, version=None): self.setup_migration_version_control() mig.upgrade(self.metadata.bind, self.migrate_repository, version=version) self.init_const_data() - + ##this prints the diffs in a readable format ##import pprint ##from migrate.versioning.schemadiff import getDiffOfModelAgainstDatabase @@ -402,7 +422,7 @@ def revision_as_dict(revision, include_packages=True, include_groups=True,ref_pa if include_groups: revision_dict['groups'] = [getattr(grp, ref_package_by) \ for grp in revision.groups if grp] - + return revision_dict def is_id(id_string): diff --git a/ckan/model/package.py b/ckan/model/package.py index 226b6841a91..e2fc2809f90 100644 --- a/ckan/model/package.py +++ b/ckan/model/package.py @@ -113,6 +113,7 @@ def get_resource_identity(resource_obj_or_dict): else: resource = resource_obj_or_dict res_dict = resource.as_dict(core_columns_only=True) + del res_dict['created'] return res_dict existing_res_identites = [get_resource_identity(res) \ for res in self.resources] @@ -169,6 +170,7 @@ def add_tag(self, tag): package_tag = model.PackageTag(self, tag) meta.Session.add(package_tag) + def add_tags(self, tags): for tag in tags: self.add_tag(tag) @@ -272,6 +274,10 @@ def as_dict(self, ref_package_by='name', ref_group_by='name'): _dict['metadata_created'] = self.metadata_created.isoformat() \ if self.metadata_created else None _dict['notes_rendered'] = ckan.misc.MarkdownFormat().to_html(self.notes) + #tracking + import ckan.model as model + tracking = model.TrackingSummary.get_for_package(self.id) + _dict['tracking_summary'] = tracking return _dict def add_relationship(self, type_, related_package, comment=u''): @@ -559,6 +565,13 @@ def metadata_modified(self): timestamp_float = timegm(timestamp_without_usecs) + usecs return datetime.datetime.utcfromtimestamp(timestamp_float) + @property + def is_private(self): + """ + A package is private if belongs to any private groups + """ + return bool(self.get_groups(capacity='private')) + def is_in_group(self, group): return group in self.get_groups() diff --git a/ckan/model/related.py b/ckan/model/related.py new file mode 100644 index 00000000000..df2d5890317 --- /dev/null +++ b/ckan/model/related.py @@ -0,0 +1,72 @@ +import os +import datetime +import meta +import sqlalchemy as sa +from core import DomainObject +from types import make_uuid +from package import Package + + +related_table = meta.Table('related',meta.metadata, + meta.Column('id', meta.UnicodeText, primary_key=True, default=make_uuid), + meta.Column('type', meta.UnicodeText, default=u'idea'), + meta.Column('title', meta.UnicodeText), + meta.Column('description', meta.UnicodeText), + meta.Column('image_url', meta.UnicodeText), + meta.Column('url', meta.UnicodeText), + meta.Column('created', meta.DateTime, default=datetime.datetime.now), + meta.Column('owner_id', meta.UnicodeText), + ) + +related_dataset_table = meta.Table('related_dataset', meta.metadata, + meta.Column('id', meta.UnicodeText, primary_key=True, default=make_uuid), + meta.Column('dataset_id', meta.UnicodeText, meta.ForeignKey('package.id'), + nullable=False), + meta.Column('related_id', meta.UnicodeText, meta.ForeignKey('related.id'), nullable=False), + meta.Column('status', meta.UnicodeText, default=u'active'), + ) + +class RelatedDataset(DomainObject): + pass + +class Related(DomainObject): + + @classmethod + def get(cls, id): + return meta.Session.query(Related).filter(Related.id == id).first() + + @classmethod + def get_for_dataset(cls, package, status=u'active'): + """ + Allows the caller to get non-active state relations between + the dataset and related, using the RelatedDataset object + """ + query = meta.Session.query(RelatedDataset).\ + filter(RelatedDataset.dataset_id==package.id).\ + filter(RelatedDataset.status==status).all() + return query + + def deactivate(self, package): + related_ds = meta.Session.query(RelatedDataset).\ + filter(RelatedDataset.dataset_id==package.id).\ + filter(RelatedDataset.status=='active').first() + if related_ds: + related_ds.status = 'inactive' + meta.Session.commit() + + +# We have avoided using SQLAlchemy association objects see +# http://bit.ly/sqlalchemy_association_object by only having the +# relation be for 'active' related objects. For non-active states +# the caller will have to use get_for_dataset() in Related. +meta.mapper(RelatedDataset, related_dataset_table, properties={ + 'related': meta.relation(Related), + 'dataset': meta.relation(Package) +}) +meta.mapper(Related, related_table, properties={ +'datasets': meta.relation(Package, + backref=meta.backref('related'), + secondary=related_dataset_table, + secondaryjoin=sa.and_(related_dataset_table.c.dataset_id==Package.id, + RelatedDataset.status=='active')) +}) diff --git a/ckan/model/resource.py b/ckan/model/resource.py index f20673ef9d0..95df2cbe9ef 100644 --- a/ckan/model/resource.py +++ b/ckan/model/resource.py @@ -14,7 +14,7 @@ from ckan.model.activity import ActivityDetail import domain_object -__all__ = ['Resource', 'resource_table', +__all__ = ['Resource', 'resource_table', 'ResourceGroup', 'resource_group_table', 'ResourceRevision', 'resource_revision_table', 'ResourceGroupRevision', 'resource_group_revision_table', @@ -22,9 +22,9 @@ CORE_RESOURCE_COLUMNS = ['url', 'format', 'description', 'hash', 'name', 'resource_type', 'mimetype', 'mimetype_inner', - 'size', 'last_modified', 'cache_url', 'cache_last_updated', - 'webstore_url', 'webstore_last_updated'] - + 'size', 'created', 'last_modified', 'cache_url', + 'cache_last_updated', 'webstore_url', + 'webstore_last_updated'] ##formally package_resource @@ -43,6 +43,7 @@ Column('mimetype', types.UnicodeText), Column('mimetype_inner', types.UnicodeText), Column('size', types.BigInteger), + Column('created', types.DateTime, default=datetime.datetime.now), Column('last_modified', types.DateTime), Column('cache_url', types.UnicodeText), Column('cache_last_updated', types.DateTime), @@ -110,6 +111,9 @@ def as_dict(self, core_columns_only=False): _dict[k] = v if self.resource_group and not core_columns_only: _dict["package_id"] = self.resource_group.package_id + import ckan.model as model + tracking = model.TrackingSummary.get_for_resource(self.url) + _dict['tracking_summary'] = tracking return _dict @classmethod diff --git a/ckan/model/tracking.py b/ckan/model/tracking.py new file mode 100644 index 00000000000..384f84bb299 --- /dev/null +++ b/ckan/model/tracking.py @@ -0,0 +1,38 @@ +from meta import * +from domain_object import DomainObject + +tracking_summary_table = Table('tracking_summary', metadata, + Column('url', UnicodeText, primary_key=True, nullable=False), + Column('package_id', UnicodeText), + Column('tracking_type', Unicode(10), nullable=False), + Column('count', Integer, nullable=False), + Column('running_total', Integer, nullable=False), + Column('recent_views', Integer, nullable=False), + Column('tracking_date', DateTime), + ) + +class TrackingSummary(DomainObject): + + @classmethod + def get_for_package(cls, package_id): + obj = Session.query(cls).autoflush(False) + obj = obj.filter_by(package_id=package_id) + data = obj.order_by('tracking_date desc').first() + if data: + return {'total' : data.running_total, + 'recent': data.recent_views} + + return {'total' : 0, 'recent' : 0} + + + @classmethod + def get_for_resource(cls, url): + obj = Session.query(cls).autoflush(False) + data = obj.filter_by(url=url).order_by('tracking_date desc').first() + if data: + return {'total' : data.running_total, + 'recent': data.recent_views} + + return {'total' : 0, 'recent' : 0} + +mapper(TrackingSummary, tracking_summary_table) diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index ee092d79ae2..bc2eb889f60 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -13,7 +13,9 @@ 'IPackageController', 'IPluginObserver', 'IConfigurable', 'IConfigurer', 'IAuthorizer', 'IActions', 'IResourceUrlChange', 'IDatasetForm', - 'IGroupForm', 'ITemplateHelpers', + 'IGroupForm', + 'ITagController', + 'ITemplateHelpers', ] from inspect import isclass @@ -181,6 +183,21 @@ class IResourceUrlChange(Interface): def notify(self, resource): pass +class ITagController(Interface): + ''' + Hook into the Tag controller. These will usually be called just before + committing or returning the respective object, i.e. all validation, + synchronization and authorization setup are complete. + + ''' + def before_view(self, tag_dict): + ''' + Extensions will recieve this before the tag gets displayed. The + dictionary passed will be the one that gets sent to the template. + + ''' + return tag_dict + class IGroupController(Interface): """ Hook into the Group controller. These will diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index 3eb898371a0..fa1f2c38cb0 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -40,7 +40,7 @@ class _Toolkit(object): 'literal', # stop tags in a string being escaped 'get_action', # get logic action function 'check_access', # check logic function authorisation - 'ActionNotFound', # action not found exception (ckan.logic.NotFound) + 'ObjectNotFound', # action not found exception (ckan.logic.NotFound) 'NotAuthorized', # action not authorized exception 'ValidationError', # model update validation error 'CkanCommand', # class for providing cli interfaces @@ -85,7 +85,7 @@ def _initialize(self): t['get_action'] = logic.get_action t['check_access'] = logic.check_access - t['ActionNotFound'] = logic.NotFound ## Name change intentional + t['ObjectNotFound'] = logic.NotFound ## Name change intentional t['NotAuthorized'] = logic.NotAuthorized t['ValidationError'] = logic.ValidationError @@ -183,6 +183,8 @@ def __getattr__(self, name): if name in self._toolkit: return self._toolkit[name] else: + if name == '__bases__': + return self.__class__.__bases__ raise Exception('`%s` not found in plugins toolkit' % name) toolkit = _Toolkit() diff --git a/ckan/public/css/style.css b/ckan/public/css/style.css index 981156e68a8..7e9e6093173 100644 --- a/ckan/public/css/style.css +++ b/ckan/public/css/style.css @@ -1,23 +1,23 @@ body.no-sidebar .sidebar-outer { display: none; } body.no-sidebar #content { padding-right: 0; border-right: none; } -body.no-sidebar .content-outer { - width: 940px; +body.no-sidebar .content-outer { + width: 940px; } .header.outer { background-color: #e2e2e2; - background-image: -webkit-gradient(linear, left top, left bottom, from(#e2e2e2), to(#cccccc)); - background-image: -webkit-linear-gradient(top, #e2e2e2, #cccccc); - background-image: -moz-linear-gradient(top, #e2e2e2, #cccccc); - background-image: -ms-linear-gradient(top, #e2e2e2, #cccccc); - background-image: -o-linear-gradient(top, #e2e2e2, #cccccc); + background-image: -webkit-gradient(linear, left top, left bottom, from(#e2e2e2), to(#cccccc)); + background-image: -webkit-linear-gradient(top, #e2e2e2, #cccccc); + background-image: -moz-linear-gradient(top, #e2e2e2, #cccccc); + background-image: -ms-linear-gradient(top, #e2e2e2, #cccccc); + background-image: -o-linear-gradient(top, #e2e2e2, #cccccc); background-image: linear-gradient(top, #e2e2e2, #cccccc); - filter: progid:DXImageTransform.Microsoft.gradient(startColorStr='#e2e2e2', EndColorStr='#cccccc'); + filter: progid:DXImageTransform.Microsoft.gradient(startColorStr='#e2e2e2', EndColorStr='#cccccc'); margin-bottom: 18px; - -moz-box-shadow: 0px 2px 15px #dddddd; - -webkit-box-shadow: 0px 2px 15px #dddddd; - box-shadow: 0px 2px 15px #dddddd; + -moz-box-shadow: 0px 2px 15px #dddddd; + -webkit-box-shadow: 0px 2px 15px #dddddd; + box-shadow: 0px 2px 15px #dddddd; border-bottom: 1px solid #ccc; } @@ -84,22 +84,22 @@ header .search { padding: 0.4em; font-weight: bold; - -moz-border-radius: 5px; - -webkit-border-radius: 5px; - border-radius: 5px; - -moz-background-clip: padding; -webkit-background-clip: padding-box; background-clip: padding-box; + -moz-border-radius: 5px; + -webkit-border-radius: 5px; + border-radius: 5px; + -moz-background-clip: padding; -webkit-background-clip: padding-box; background-clip: padding-box; } .footer.outer { border-top: 2px solid #ccc; background-color: #dbdbdb; - background-image: -webkit-gradient(linear, left top, left bottom, from(#dbdbdb), to(#ffffff)); - background-image: -webkit-linear-gradient(top, #dbdbdb, #ffffff); - background-image: -moz-linear-gradient(top, #dbdbdb, #ffffff); - background-image: -ms-linear-gradient(top, #dbdbdb, #ffffff); - background-image: -o-linear-gradient(top, #dbdbdb, #ffffff); + background-image: -webkit-gradient(linear, left top, left bottom, from(#dbdbdb), to(#ffffff)); + background-image: -webkit-linear-gradient(top, #dbdbdb, #ffffff); + background-image: -moz-linear-gradient(top, #dbdbdb, #ffffff); + background-image: -ms-linear-gradient(top, #dbdbdb, #ffffff); + background-image: -o-linear-gradient(top, #dbdbdb, #ffffff); background-image: linear-gradient(top, #dbdbdb, #ffffff); - fromilter: progid:DXImageTransform.Microsoft.gradient(startColorStr='#dbdbdb', EndColorStr='#ffffff'); + fromilter: progid:DXImageTransform.Microsoft.gradient(startColorStr='#dbdbdb', EndColorStr='#ffffff'); } html, body { @@ -134,7 +134,7 @@ footer h3 { } h1, h2, h3, h4, h5 { - font-family: 'Ubuntu', Georgia; + font-family: 'Ubuntu', Georgia; font-weight: normal; margin-bottom: 10px; } @@ -148,6 +148,10 @@ a:hover { color: #183661; } +a.btn-primary:visited { + color: #fff; +} + label.control-label { font-weight: bold; } @@ -160,16 +164,16 @@ label.control-label { table th { border: 1px solid #e0e0e0; background-color: #e2e2e2; - background-image: -webkit-gradient(linear, left top, left bottom, from(#f0f0f0), to(#e2e2e2)); - background-image: -webkit-linear-gradient(top, #f0f0f0, #e2e2e2); - background-image: -moz-linear-gradient(top, #f0f0f0, #e2e2e2); - background-image: -ms-linear-gradient(top, #f0f0f0, #e2e2e2); - background-image: -o-linear-gradient(top, #f0f0f0, #e2e2e2); + background-image: -webkit-gradient(linear, left top, left bottom, from(#f0f0f0), to(#e2e2e2)); + background-image: -webkit-linear-gradient(top, #f0f0f0, #e2e2e2); + background-image: -moz-linear-gradient(top, #f0f0f0, #e2e2e2); + background-image: -ms-linear-gradient(top, #f0f0f0, #e2e2e2); + background-image: -o-linear-gradient(top, #f0f0f0, #e2e2e2); background-image: linear-gradient(top, #f0f0f0, #e2e2e2); filter: progid:DXImageTransform.Microsoft.gradient(startColorStr='#f0f0f0', EndColorStr='#e2e2e2'); } table caption { - caption-side: bottom; + caption-side: bottom; color: #888; font-size: 0.9em; background-color: white; @@ -241,10 +245,10 @@ img.gravatar { vertical-align: top; } -.drag-drop-list { - list-style-type: none; - margin: 0; - padding: 0; +.drag-drop-list { + list-style-type: none; + margin: 0; + padding: 0; } .drag-drop-list li { margin-bottom: 3px; @@ -269,18 +273,18 @@ ul.no-break li { margin: 0 0 1em 0 ; border: 1px solid #e0e0e0; background-color: #e2e2e2; - background-image: -webkit-gradient(linear, left top, left bottom, from(#f0f0f0), to(#e2e2e2)); - background-image: -webkit-linear-gradient(top, #f0f0f0, #e2e2e2); - background-image: -moz-linear-gradient(top, #f0f0f0, #e2e2e2); - background-image: -ms-linear-gradient(top, #f0f0f0, #e2e2e2); - background-image: -o-linear-gradient(top, #f0f0f0, #e2e2e2); + background-image: -webkit-gradient(linear, left top, left bottom, from(#f0f0f0), to(#e2e2e2)); + background-image: -webkit-linear-gradient(top, #f0f0f0, #e2e2e2); + background-image: -moz-linear-gradient(top, #f0f0f0, #e2e2e2); + background-image: -ms-linear-gradient(top, #f0f0f0, #e2e2e2); + background-image: -o-linear-gradient(top, #f0f0f0, #e2e2e2); background-image: linear-gradient(top, #f0f0f0, #e2e2e2); filter: progid:DXImageTransform.Microsoft.gradient(startColorStr='#f0f0f0', EndColorStr='#e2e2e2'); - -moz-border-radius: 5px; - -webkit-border-radius: 5px; - border-radius: 5px; - -moz-background-clip: padding; -webkit-background-clip: padding-box; background-clip: padding-box; + -moz-border-radius: 5px; + -webkit-border-radius: 5px; + border-radius: 5px; + -moz-background-clip: padding; -webkit-background-clip: padding-box; background-clip: padding-box; padding: 2px 4px; font-weight: bold; } @@ -348,7 +352,7 @@ ul.no-break li { #sidebar { overflow: hidden; } -#sidebar h2, +#sidebar h2, #sidebar h3 { font-size: 1.3em; } @@ -366,9 +370,9 @@ ul.no-break li { background-color: #FFF7C0; padding: 15px; padding-top: 10px; - -moz-border-radius: 15px; - -webkit-border-radius: 15px; - border-radius: 15px; + -moz-border-radius: 15px; + -webkit-border-radius: 15px; + border-radius: 15px; } @@ -411,7 +415,7 @@ ul.no-break li { } .facet-box .facet-options li { padding-top: 0.2em; - color: #000; + color: #000; } @@ -539,7 +543,7 @@ body.index.home .front-page .group strong { /* = Login Form = */ /* ============== */ form.simple-form label { - display: inline-block; + display: inline-block; float: left; min-width: 40%; padding-top: 0.5em; @@ -548,7 +552,7 @@ form.simple-form label { form.simple-form input[type=text], form.simple-form input[type=password] { border: 1px solid #E7E7E7; - padding: 0.5em; + padding: 0.5em; width: 40%; margin-bottom: 1em } @@ -593,7 +597,7 @@ form.simple-form input[type=password] { /* = User Index = */ /* ============== */ -ul.userlist, +ul.userlist, ul.userlist ul { list-style-type: none; margin: 0; @@ -631,9 +635,9 @@ ul.userlist .badge { /* ================== */ body.user.read #sidebar { display: none; } -body.user.read #content { - border-right: 0; - width: 950px; +body.user.read #content { + border-right: 0; + width: 950px; } .user.read .page_heading { @@ -738,10 +742,10 @@ input.search { border: 1px solid #ccc; padding: 0.5em; font-weight: bold; - -moz-border-radius: 5px; - -webkit-border-radius: 5px; - border-radius: 5px; - -moz-background-clip: padding; -webkit-background-clip: padding-box; background-clip: padding-box; + -moz-border-radius: 5px; + -webkit-border-radius: 5px; + border-radius: 5px; + -moz-background-clip: padding; -webkit-background-clip: padding-box; background-clip: padding-box; } .dataset-search input.button { display: inline-block; @@ -785,7 +789,7 @@ ul.datasets .search_meta { } ul.datasets ul.dataset_formats { float: right; - padding: 0 0 3px 0; + padding: 0 0 3px 0; margin: 0; font-family: monospace; } @@ -810,7 +814,7 @@ ul.datasets .openness img { ul.datasets .openness li { margin:0; padding:0; - border:none; + border:none; } @@ -1018,14 +1022,14 @@ body.package.read .related-datasets li { } img.open-data { margin: 1px 0 0 8px; vertical-align: top; } -#dataset-resources { - margin-top: 2em; - margin-bottom: 2em; +#dataset-resources { + margin-top: 2em; + margin-bottom: 2em; } body.package.read h3 { margin-bottom: 8px; } -.search-result { +.search-result { border-left: 2px solid #eee; margin: 0; padding: 8px; @@ -1059,14 +1063,14 @@ body.package.read h3 { } .search-result .result-url, -.search-result .result-url a { +.search-result .result-url a { color: #888; } .search-result .result-lrl:hover, -.search-result .result-url:hover a { +.search-result .result-url:hover a { color: #333; } -.search-result .result-url:hover a { +.search-result .result-url:hover a { text-decoration: underline; } .search-result .result-url img { @@ -1137,9 +1141,9 @@ body.package.read #sidebar li.widget-container { /* ====================== */ body.package.resource_read #sidebar { display: none; } -body.package.resource_read #content { - border-right: 0; - width: 950px; +body.package.resource_read #content { + border-right: 0; + width: 950px; } .resource_read .notes { @@ -1185,7 +1189,7 @@ body.about #content { border-right: 0; } /* ============== */ /* = Admin Page = */ /* ============== */ -body.admin form#form-purge-packages, +body.admin form#form-purge-packages, body.admin form#form-purge-revisions { margin-bottom: 30px; text-align: right; @@ -1403,3 +1407,73 @@ body.editresources .error-explanation { /* Let JS render the resource errors inline */ display: none; } + +/* Modal Dialog Styles */ + +.modal-header .heading { + margin-bottom: 0; +} + +.modal-body form { + margin-bottom: 0; +} + +/* Chosen Form Styles */ + +.chzn-container-single { + margin-bottom: 9px; /* Keep Chosen inline with Bootstrap */ +} + +.form-inline .chzn-container-single, +.form-horizontal .chzn-select { + margin-bottom: 0; +} + +.required { + color: #808080; +} + +.thumbnails li:nth-of-type(5n) { + clear: left; +} + +.thumbnail .heading { + font-weight: bold; +} + +.thumbnail .image { + display: block; + width: 210px; + height: 180px; + overflow: hidden; + background: #ececec; + border: 1px solid #dedede; + margin: -1px; +} + +.thumbnail .image img { + max-width: 100%; + width: auto; + height: auto; +} + +.thumbnail .empty { + color: #ccc; +} + +.thumbnail .read-more { + margin-top: 10px; + margin-bottom: 0; +} + +.no-related-items { + font-size: 16px; + color: #666; + background: #EBEBEB; + border: 1px solid #DBDBDB; + padding: 20px; + border-radius: 5px; + margin: 40px auto; + float: none; + text-align: center; +} diff --git a/ckan/public/images/photo-placeholder.png b/ckan/public/images/photo-placeholder.png new file mode 100644 index 00000000000..efdea70ea20 Binary files /dev/null and b/ckan/public/images/photo-placeholder.png differ diff --git a/ckan/public/scripts/application.js b/ckan/public/scripts/application.js index 995e7033362..0d3628ebf76 100644 --- a/ckan/public/scripts/application.js +++ b/ckan/public/scripts/application.js @@ -9,6 +9,7 @@ CKAN.Utils = CKAN.Utils || {}; /* ================================= */ (function ($) { $(document).ready(function () { + CKAN.Utils.relatedSetup($("#form-add-related")); CKAN.Utils.setupUserAutocomplete($('input.autocomplete-user')); CKAN.Utils.setupOrganizationUserAutocomplete($('input.autocomplete-organization-user')); CKAN.Utils.setupGroupAutocomplete($('input.autocomplete-group')); @@ -44,6 +45,12 @@ CKAN.Utils = CKAN.Utils || {}; if (isResourceView) { CKAN.DataPreview.loadPreviewDialog(preload_resource); } + + var isEmbededDataviewer = $('body.package.resource_embedded_dataviewer').length > 0; + if (isEmbededDataviewer) { + CKAN.DataPreview.loadEmbeddedPreview(preload_resource, reclineState); + } + var isDatasetNew = $('body.package.new').length > 0; if (isDatasetNew) { // Set up magic URL slug editor @@ -119,6 +126,32 @@ CKAN.Utils = CKAN.Utils || {}; }); }(jQuery)); +/* =============================== */ +/* jQuery Plugins */ +/* =============================== */ + +jQuery.fn.truncate = function (max, suffix) { + return this.each(function () { + var element = jQuery(this), + cached = element.text(), + length = max || element.data('truncate') || 30, + text = cached.slice(0, length), + expand = jQuery('').text(suffix || '»'); + + // Try to truncate to nearest full word. + while ((/\S/).test(text[text.length - 1])) { + text = text.slice(0, text.length - 1); + } + + element.html(jQuery.trim(text)); + + expand.appendTo(element.append(' ')); + expand.click(function (event) { + event.preventDefault(); + element.text(cached); + }); + }); +}; /* =============================== */ /* Backbone Model: Resource object */ @@ -638,6 +671,7 @@ CKAN.View.Resource = Backbone.View.extend({ word=='format' || word=='hash' || word=='id' || + word=='created' || word=='last_modified' || word=='mimetype' || word=='mimetype_inner' || @@ -692,7 +726,7 @@ CKAN.View.ResourceAddUpload = Backbone.View.extend({ setupFileUpload: function() { var self = this; this.el.find('.fileupload').fileupload({ - // needed because we are posting to remote url + // needed because we are posting to remote url forceIframeTransport: true, replaceFileInput: false, autoUpload: false, @@ -725,7 +759,7 @@ CKAN.View.ResourceAddUpload = Backbone.View.extend({ }, // Create an upload key/label for this file. - // + // // Form: {current-date}/file-name. Do not just use the file name as this // would lead to collisions. // (Could add userid/username and/or a small random string to reduce @@ -790,7 +824,7 @@ CKAN.View.ResourceAddUpload = Backbone.View.extend({ newResource.set({ url: data._location , name: name - , size: data._content_length + , size: data._content_length , last_modified: lastmod , format: data._format , mimetype: data._format @@ -873,7 +907,7 @@ CKAN.View.ResourceAddUrl = Backbone.View.extend({ self.resetForm(); } }); - } + } else { newResource.set({url: urlVal, resource_type: this.options.mode}); if (newResource.get('resource_type')=='file') { @@ -958,7 +992,6 @@ CKAN.Utils = function($, my) { , select: function(event, ui) { var input_box = $(this); input_box.val(''); - var parent_dd = input_box.parent('dd'); var old_name = input_box.attr('name'); var field_name_regex = /^(\S+)__(\d+)__(\S+)$/; var split = old_name.match(field_name_regex); @@ -967,10 +1000,15 @@ CKAN.Utils = function($, my) { input_box.attr('name', new_name); input_box.attr('id', new_name); - - parent_dd.before( - '' + '
' + ui.item.label + '
' - ); + + var $new = $('

'); + $new.append($('').attr('name', old_name).val(ui.item.value)); + $new.append(' '); + $new.append(ui.item.label); + input_box.after($new); + + // prevent setting value in autocomplete box + return false; } }); }; @@ -1101,6 +1139,68 @@ CKAN.Utils = function($, my) { }); }; + + my.relatedSetup = function(form) { + function addAlert(msg) { + $('
').html(msg).hide().prependTo(form).fadeIn(); + } + + // Center thumbnails vertically. + $('.related-items').each(function () { + var item = $(this); + + function vertiallyAlign() { + var img = $(this), + height = img.height(), + parent = img.parent().height(), + top = (height - parent) / 2; + + if (parent < height) { + img.css('margin-top', -top); + } + } + + item.find('img').load(vertiallyAlign); + item.find('.description').truncate(); + }); + + $(form).submit(function (event) { + event.preventDefault(); + + // Validate the form + var form = $(this), data = {}; + jQuery.each(form.serializeArray(), function () { + data[this.name] = this.value; + }); + + form.find('.alert').remove(); + form.find('.error').removeClass('error'); + if (!data.title) { + addAlert('Missing field: A title is required'); + $('[name=title]').parent().addClass('error'); + return; + } + if (!data.url) { + addAlert('Missing field: A url is required'); + $('[name=url]').parent().addClass('error'); + return; + } + + $.ajax({ + type: this.method, + url: CKAN.SITE_URL + '/api/3/action/related_create', + data: JSON.stringify(data), + success: function (data) { + window.location.reload(); + }, + error: function(err, txt, w) { + // This needs to be far more informative. + addAlert('Error: Unable to add related item'); + } + }); + }); + }; + // Attach authz group autocompletion to provided elements // // Requires: jquery-ui autocomplete @@ -1250,6 +1350,81 @@ CKAN.DataPreview = function ($, my) { my.dialogId = 'ckanext-datapreview'; my.$dialog = $('#' + my.dialogId); + // **Public: Loads a data previewer for an embedded page** + // + // Uses the provided reclineState to restore the Dataset. Creates a single + // view for the Dataset (the one defined by reclineState.currentView). And + // then passes the constructed Dataset, the constructed View, and the + // reclineState into the DataExplorer constructor. + my.loadEmbeddedPreview = function(resourceData, reclineState) { + my.$dialog.html('

Loading ...

'); + + // Restore the Dataset from the given reclineState. + var dataset = recline.Model.Dataset.restore(reclineState); + + // Only create the view defined in reclineState.currentView. + // TODO: tidy this up. + var views = null; + if (reclineState.currentView === 'grid') { + views = [ { + id: 'grid', + label: 'Grid', + view: new recline.View.Grid({ + model: dataset, + state: reclineState['view-grid'] + }) + }]; + } else if (reclineState.currentView === 'graph') { + views = [ { + id: 'graph', + label: 'Graph', + view: new recline.View.Graph({ + model: dataset, + state: reclineState['view-graph'] + }) + }]; + } else if (reclineState.currentView === 'map') { + views = [ { + id: 'map', + label: 'Map', + view: new recline.View.Map({ + model: dataset, + state: reclineState['view-map'] + }) + }]; + } + + // Finally, construct the DataExplorer. Again, passing in the reclineState. + var dataExplorer = new recline.View.DataExplorer({ + el: my.$dialog, + model: dataset, + state: reclineState, + views: views + }); + + Backbone.history.start(); + }; + + // **Public: Creates a link to the embeddable page. + // + // For a given DataExplorer state, this function constructs and returns the + // url to the embeddable view of the current dataexplorer state. + my.makeEmbedLink = function(explorerState) { + var state = explorerState.toJSON(); + state.state_version = 1; + + var queryString = '?'; + var items = []; + $.each(state, function(key, value) { + if (typeof(value) === 'object') { + value = JSON.stringify(value); + } + items.push(key + '=' + escape(value)); + }); + queryString += items.join('&'); + return embedPath + queryString; + }; + // **Public: Loads a data preview** // // Fetches the preview data object from the link provided and loads the @@ -1267,14 +1442,14 @@ CKAN.DataPreview = function ($, my) { { id: 'grid', label: 'Grid', - view: new recline.View.DataGrid({ + view: new recline.View.Grid({ model: dataset }) }, { id: 'graph', label: 'Graph', - view: new recline.View.FlotGraph({ + view: new recline.View.Graph({ model: dataset }) }, @@ -1294,6 +1469,58 @@ CKAN.DataPreview = function ($, my) { readOnly: true } }); + + // ----------------------------- + // Setup the Embed modal dialog. + // ----------------------------- + + // embedLink holds the url to the embeddable view of the current DataExplorer state. + var embedLink = $('.embedLink'); + + // embedIframeText contains the '', + { + link: link.replace(/"/g, '"'), + width: width, + height: height + })); + embedLink.attr('href', link); + } + + // Bind changes to the DataExplorer, or the two width and height inputs + // to re-calculate the url. + dataExplorer.state.bind('change', updateLink); + for (var i=0; i \ +
\ + \ +
\ +
\
\ A Document (aka Row) // // A single entry or row in the dataset @@ -211,7 +285,8 @@ my.DocumentList = Backbone.Collection.extend({ // * format: (optional) used to indicate how the data should be formatted. For example: // * type=date, format=yyyy-mm-dd // * type=float, format=percentage -// * type=float, format='###,###.##' +// * type=string, format=link (render as hyperlink) +// * type=string, format=markdown (render as markdown if Showdown available) // * is_derived: (default: false) attribute indicating this field has no backend data but is just derived from other fields (see below). // // Following additional instance properties: @@ -267,6 +342,22 @@ my.Field = Backbone.Model.extend({ if (format === 'percentage') { return val + '%'; } + return val; + }, + 'string': function(val, field, doc) { + var format = field.get('format'); + if (format === 'link') { + return 'VAL'.replace(/VAL/g, val); + } else if (format === 'markdown') { + if (typeof Showdown !== 'undefined') { + var showdown = new Showdown.converter(); + out = showdown.makeHtml(val); + return out; + } else { + return val; + } + } + return val; } } }); @@ -449,6 +540,13 @@ my.FacetList = Backbone.Collection.extend({ model: my.Facet }); +// ## Object State +// +// Convenience Backbone model for storing (configuration) state of objects like Views. +my.ObjectState = Backbone.Model.extend({ +}); + + // ## Backend registry // // Backends will register themselves by id into this registry @@ -618,10 +716,10 @@ this.recline.View = this.recline.View || {}; // ## Graph view for a Dataset using Flot graphing library. // -// Initialization arguments: +// Initialization arguments (in a hash in first parameter): // // * model: recline.Model.Dataset -// * config: (optional) graph configuration hash of form: +// * state: (optional) configuration hash of form: // // { // group: {column name for x-axis}, @@ -631,10 +729,10 @@ this.recline.View = this.recline.View || {}; // // NB: should *not* provide an el argument to the view but must let the view // generate the element itself (you can then append view.el to the DOM. -my.FlotGraph = Backbone.View.extend({ +my.Graph = Backbone.View.extend({ tagName: "div", - className: "data-graph-container", + className: "recline-graph", template: ' \
\ @@ -697,7 +795,7 @@ my.FlotGraph = Backbone.View.extend({ 'click .action-toggle-help': 'toggleHelp' }, - initialize: function(options, config) { + initialize: function(options) { var self = this; this.el = $(this.el); _.bindAll(this, 'render', 'redraw'); @@ -707,18 +805,14 @@ my.FlotGraph = Backbone.View.extend({ this.model.fields.bind('add', this.render); this.model.currentDocuments.bind('add', this.redraw); this.model.currentDocuments.bind('reset', this.redraw); - var configFromHash = my.parseHashQueryString().graph; - if (configFromHash) { - configFromHash = JSON.parse(configFromHash); - } - this.chartConfig = _.extend({ + var stateData = _.extend({ group: null, series: [], graphType: 'lines-and-points' }, - configFromHash, - config - ); + options.state + ); + this.state = new recline.Model.ObjectState(stateData); this.render(); }, @@ -740,13 +834,12 @@ my.FlotGraph = Backbone.View.extend({ var series = this.$series.map(function () { return $(this).val(); }); - this.chartConfig.series = $.makeArray(series); - this.chartConfig.group = this.el.find('.editor-group select').val(); - this.chartConfig.graphType = this.el.find('.editor-type select').val(); - // update navigation - var qs = my.parseHashQueryString(); - qs.graph = JSON.stringify(this.chartConfig); - my.setHashQueryString(qs); + var updatedState = { + series: $.makeArray(series), + group: this.el.find('.editor-group select').val(), + graphType: this.el.find('.editor-type select').val() + }; + this.state.set(updatedState); this.redraw(); }, @@ -762,7 +855,7 @@ my.FlotGraph = Backbone.View.extend({ return; } var series = this.createSeries(); - var options = this.getGraphOptions(this.chartConfig.graphType); + var options = this.getGraphOptions(this.state.attributes.graphType); this.plot = $.plot(this.$graph, series, options); this.setupTooltips(); // create this.plot and cache it @@ -783,7 +876,7 @@ my.FlotGraph = Backbone.View.extend({ // special tickformatter to show labels rather than numbers var tickFormatter = function (val) { if (self.model.currentDocuments.models[val]) { - var out = self.model.currentDocuments.models[val].get(self.chartConfig.group); + var out = self.model.currentDocuments.models[val].get(self.state.attributes.group); // if the value was in fact a number we want that not the if (typeof(out) == 'number') { return val; @@ -866,14 +959,14 @@ my.FlotGraph = Backbone.View.extend({ var y = item.datapoint[1]; // convert back from 'index' value on x-axis (e.g. in cases where non-number values) if (self.model.currentDocuments.models[x]) { - x = self.model.currentDocuments.models[x].get(self.chartConfig.group); + x = self.model.currentDocuments.models[x].get(self.state.attributes.group); } else { x = x.toFixed(2); } y = y.toFixed(2); var content = _.template('<%= group %> = <%= x %>, <%= series %> = <%= y %>', { - group: self.chartConfig.group, + group: self.state.attributes.group, x: x, series: item.series.label, y: y @@ -891,25 +984,23 @@ my.FlotGraph = Backbone.View.extend({ createSeries: function () { var self = this; var series = []; - if (this.chartConfig) { - $.each(this.chartConfig.series, function (seriesIndex, field) { - var points = []; - $.each(self.model.currentDocuments.models, function (index, doc) { - var x = doc.get(self.chartConfig.group); - var y = doc.get(field); - if (typeof x === 'string') { - x = index; - } - // horizontal bar chart - if (self.chartConfig.graphType == 'bars') { - points.push([y, x]); - } else { - points.push([x, y]); - } - }); - series.push({data: points, label: field}); + _.each(this.state.attributes.series, function(field) { + var points = []; + _.each(self.model.currentDocuments.models, function(doc, index) { + var x = doc.get(self.state.attributes.group); + var y = doc.get(field); + if (typeof x === 'string') { + x = index; + } + // horizontal bar chart + if (self.state.attributes.graphType == 'bars') { + points.push([y, x]); + } else { + points.push([x, y]); + } }); - } + series.push({data: points, label: field}); + }); return series; }, @@ -969,12 +1060,12 @@ this.recline = this.recline || {}; this.recline.View = this.recline.View || {}; (function($, my) { -// ## DataGrid +// ## (Data) Grid Dataset View // // Provides a tabular view on a Dataset. // // Initialize it with a `recline.Model.Dataset`. -my.DataGrid = Backbone.View.extend({ +my.Grid = Backbone.View.extend({ tagName: "div", className: "recline-grid-container", @@ -985,12 +1076,16 @@ my.DataGrid = Backbone.View.extend({ this.model.currentDocuments.bind('add', this.render); this.model.currentDocuments.bind('reset', this.render); this.model.currentDocuments.bind('remove', this.render); - this.state = {}; - this.hiddenFields = []; + this.tempState = {}; + var state = _.extend({ + hiddenFields: [] + }, modelEtc.state + ); + this.state = new recline.Model.ObjectState(state); }, events: { - 'click .column-header-menu': 'onColumnHeaderClick', + 'click .column-header-menu .data-table-menu li a': 'onColumnHeaderClick', 'click .row-header-menu': 'onRowHeaderClick', 'click .root-header-menu': 'onRootHeaderClick', 'click .data-table-menu li a': 'onMenuClick' @@ -1012,11 +1107,11 @@ my.DataGrid = Backbone.View.extend({ // Column and row menus onColumnHeaderClick: function(e) { - this.state.currentColumn = $(e.target).closest('.column-header').attr('data-field'); + this.tempState.currentColumn = $(e.target).closest('.column-header').attr('data-field'); }, onRowHeaderClick: function(e) { - this.state.currentRow = $(e.target).parents('tr:first').attr('data-id'); + this.tempState.currentRow = $(e.target).parents('tr:first').attr('data-id'); }, onRootHeaderClick: function(e) { @@ -1024,7 +1119,7 @@ my.DataGrid = Backbone.View.extend({ {{#columns}} \
  • Show column: {{.}}
  • \ {{/columns}}'; - var tmp = $.mustache(tmpl, {'columns': this.hiddenFields}); + var tmp = $.mustache(tmpl, {'columns': this.state.get('hiddenFields')}); this.el.find('.root-header-menu .dropdown-menu').html(tmp); }, @@ -1032,15 +1127,15 @@ my.DataGrid = Backbone.View.extend({ var self = this; e.preventDefault(); var actions = { - bulkEdit: function() { self.showTransformColumnDialog('bulkEdit', {name: self.state.currentColumn}); }, + bulkEdit: function() { self.showTransformColumnDialog('bulkEdit', {name: self.tempState.currentColumn}); }, facet: function() { - self.model.queryState.addFacet(self.state.currentColumn); + self.model.queryState.addFacet(self.tempState.currentColumn); }, facet_histogram: function() { - self.model.queryState.addHistogramFacet(self.state.currentColumn); + self.model.queryState.addHistogramFacet(self.tempState.currentColumn); }, filter: function() { - self.model.queryState.addTermFilter(self.state.currentColumn, ''); + self.model.queryState.addTermFilter(self.tempState.currentColumn, ''); }, transform: function() { self.showTransformDialog('transform'); }, sortAsc: function() { self.setColumnSort('asc'); }, @@ -1051,7 +1146,7 @@ my.DataGrid = Backbone.View.extend({ var doc = _.find(self.model.currentDocuments.models, function(doc) { // important this is == as the currentRow will be string (as comes // from DOM) while id may be int - return doc.id == self.state.currentRow; + return doc.id == self.tempState.currentRow; }); doc.destroy().then(function() { self.model.currentDocuments.remove(doc); @@ -1070,7 +1165,7 @@ my.DataGrid = Backbone.View.extend({ var view = new my.ColumnTransform({ model: this.model }); - view.state = this.state; + view.state = this.tempState; view.render(); $el.empty(); $el.append(view.el); @@ -1096,17 +1191,22 @@ my.DataGrid = Backbone.View.extend({ setColumnSort: function(order) { var sort = [{}]; - sort[0][this.state.currentColumn] = {order: order}; + sort[0][this.tempState.currentColumn] = {order: order}; this.model.query({sort: sort}); }, hideColumn: function() { - this.hiddenFields.push(this.state.currentColumn); + var hiddenFields = this.state.get('hiddenFields'); + hiddenFields.push(this.tempState.currentColumn); + this.state.set({hiddenFields: hiddenFields}); + // change event not being triggered (because it is an array?) so trigger manually + this.state.trigger('change'); this.render(); }, showColumn: function(e) { - this.hiddenFields = _.without(this.hiddenFields, $(e.target).data('column')); + var hiddenFields = _.without(this.state.get('hiddenFields'), $(e.target).data('column')); + this.state.set({hiddenFields: hiddenFields}); this.render(); }, @@ -1162,41 +1262,41 @@ my.DataGrid = Backbone.View.extend({ render: function() { var self = this; this.fields = this.model.fields.filter(function(field) { - return _.indexOf(self.hiddenFields, field.id) == -1; + return _.indexOf(self.state.get('hiddenFields'), field.id) == -1; }); var htmls = $.mustache(this.template, this.toTemplateJSON()); this.el.html(htmls); this.model.currentDocuments.forEach(function(doc) { var tr = $(''); self.el.find('tbody').append(tr); - var newView = new my.DataGridRow({ + var newView = new my.GridRow({ model: doc, el: tr, fields: self.fields }); newView.render(); }); - this.el.toggleClass('no-hidden', (self.hiddenFields.length === 0)); + this.el.toggleClass('no-hidden', (self.state.get('hiddenFields').length === 0)); return this; } }); -// ## DataGridRow View for rendering an individual document. +// ## GridRow View for rendering an individual document. // // Since we want this to update in place it is up to creator to provider the element to attach to. // -// In addition you *must* pass in a FieldList in the constructor options. This should be list of fields for the DataGrid. +// In addition you *must* pass in a FieldList in the constructor options. This should be list of fields for the Grid. // // Example: // //
    -// var row = new DataGridRow({
    +// var row = new GridRow({
     //   model: dataset-document,
     //     el: dom-element,
     //     fields: mydatasets.fields // a FieldList object
     //   });
     // 
    -my.DataGridRow = Backbone.View.extend({ +my.GridRow = Backbone.View.extend({ initialize: function(initData) { _.bindAll(this, 'render'); this._fields = initData.fields; @@ -1301,21 +1401,21 @@ this.recline.View = this.recline.View || {}; // [GeoJSON](http://geojson.org) objects or two fields with latitude and // longitude coordinates. // -// Initialization arguments: -// -// * options: initial options. They must contain a model: -// -// { -// model: {recline.Model.Dataset} -// } -// -// * config: (optional) map configuration hash (not yet used) -// +// Initialization arguments are as standard for Dataset Views. State object may +// have the following (optional) configuration options: // +//
    +//   {
    +//     // geomField if specified will be used in preference to lat/lon
    +//     geomField: {id of field containing geometry in the dataset}
    +//     lonField: {id of field containing longitude in the dataset}
    +//     latField: {id of field containing latitude in the dataset}
    +//   }
    +// 
    my.Map = Backbone.View.extend({ tagName: 'div', - className: 'data-map-container', + className: 'recline-map', template: ' \
    \ @@ -1364,6 +1464,11 @@ my.Map = Backbone.View.extend({
    \ \
    \ +
    \ + \ +
    \ \
    \ \ @@ -1381,17 +1486,16 @@ my.Map = Backbone.View.extend({ // Define here events for UI elements events: { 'click .editor-update-map': 'onEditorSubmit', - 'change .editor-field-type': 'onFieldTypeChange' + 'change .editor-field-type': 'onFieldTypeChange', + 'change #editor-auto-zoom': 'onAutoZoomChange' }, - - initialize: function(options, config) { + initialize: function(options) { var self = this; - this.el = $(this.el); // Listen to changes in the fields - this.model.bind('change', function() { + this.model.fields.bind('change', function() { self._setupGeometryField(); }); this.model.fields.bind('add', this.render); @@ -1402,17 +1506,40 @@ my.Map = Backbone.View.extend({ // Listen to changes in the documents this.model.currentDocuments.bind('add', function(doc){self.redraw('add',doc)}); + this.model.currentDocuments.bind('change', function(doc){ + self.redraw('remove',doc); + self.redraw('add',doc); + }); this.model.currentDocuments.bind('remove', function(doc){self.redraw('remove',doc)}); this.model.currentDocuments.bind('reset', function(){self.redraw('reset')}); - // If the div was hidden, Leaflet needs to recalculate some sizes - // to display properly this.bind('view:show',function(){ + // If the div was hidden, Leaflet needs to recalculate some sizes + // to display properly + if (self.map){ self.map.invalidateSize(); + if (self._zoomPending && self.autoZoom) { + self._zoomToFeatures(); + self._zoomPending = false; + } + } + self.visible = true; + }); + this.bind('view:hide',function(){ + self.visible = false; }); - this.mapReady = false; + var stateData = _.extend({ + geomField: null, + lonField: null, + latField: null + }, + options.state + ); + this.state = new recline.Model.ObjectState(stateData); + this.autoZoom = true; + this.mapReady = false; this.render(); }, @@ -1429,12 +1556,12 @@ my.Map = Backbone.View.extend({ this.$map = this.el.find('.panel.map'); if (this.geomReady && this.model.fields.length){ - if (this._geomFieldName){ - this._selectOption('editor-geom-field',this._geomFieldName); + if (this.state.get('geomField')){ + this._selectOption('editor-geom-field',this.state.get('geomField')); $('#editor-field-type-geom').attr('checked','checked').change(); } else{ - this._selectOption('editor-lon-field',this._lonFieldName); - this._selectOption('editor-lat-field',this._latFieldName); + this._selectOption('editor-lon-field',this.state.get('lonField')); + this._selectOption('editor-lat-field',this.state.get('latField')); $('#editor-field-type-latlon').attr('checked','checked').change(); } } @@ -1463,9 +1590,7 @@ my.Map = Backbone.View.extend({ // * refresh: Clear existing features and add all current documents // redraw: function(action,doc){ - var self = this; - action = action || 'refresh'; if (this.geomReady && this.mapReady){ @@ -1479,6 +1604,13 @@ my.Map = Backbone.View.extend({ this.features.clearLayers(); this._add(this.model.currentDocuments.models); } + if (action != 'reset' && this.autoZoom){ + if (this.visible){ + this._zoomToFeatures(); + } else { + this._zoomPending = true; + } + } } }, @@ -1494,14 +1626,19 @@ my.Map = Backbone.View.extend({ onEditorSubmit: function(e){ e.preventDefault(); if ($('#editor-field-type-geom').attr('checked')){ - this._geomFieldName = $('.editor-geom-field > select > option:selected').val(); - this._latFieldName = this._lonFieldName = false; + this.state.set({ + geomField: $('.editor-geom-field > select > option:selected').val(), + lonField: null, + latField: null + }); } else { - this._geomFieldName = false; - this._latFieldName = $('.editor-lat-field > select > option:selected').val(); - this._lonFieldName = $('.editor-lon-field > select > option:selected').val(); + this.state.set({ + geomField: null, + lonField: $('.editor-lon-field > select > option:selected').val(), + latField: $('.editor-lat-field > select > option:selected').val() + }); } - this.geomReady = (this._geomFieldName || (this._latFieldName && this._lonFieldName)); + this.geomReady = (this.state.get('geomField') || (this.state.get('latField') && this.state.get('lonField'))); this.redraw(); return false; @@ -1520,6 +1657,10 @@ my.Map = Backbone.View.extend({ } }, + onAutoZoomChange: function(e){ + this.autoZoom = !this.autoZoom; + }, + // Private: Add one or n features to the map // // For each document passed, a GeoJSON geometry will be extracted and added @@ -1534,9 +1675,12 @@ my.Map = Backbone.View.extend({ if (!(docs instanceof Array)) docs = [docs]; + var count = 0; + var wrongSoFar = 0; _.every(docs,function(doc){ + count += 1; var feature = self._getGeometryFromDocument(doc); - if (typeof feature === 'undefined'){ + if (typeof feature === 'undefined' || feature === null){ // Empty field return true; } else if (feature instanceof Object){ @@ -1544,7 +1688,9 @@ my.Map = Backbone.View.extend({ // TODO: mustache? html = '' for (key in doc.attributes){ - html += '
    ' + key + ': '+ doc.attributes[key] + '
    ' + if (!(self.state.get('geomField') && key == self.state.get('geomField'))){ + html += '
    ' + key + ': '+ doc.attributes[key] + '
    '; + } } feature.properties = {popupContent: html}; @@ -1553,16 +1699,20 @@ my.Map = Backbone.View.extend({ feature.properties.cid = doc.cid; try { - self.features.addGeoJSON(feature); + self.features.addGeoJSON(feature); } catch (except) { - var msg = 'Wrong geometry value'; - if (except.message) msg += ' (' + except.message + ')'; + wrongSoFar += 1; + var msg = 'Wrong geometry value'; + if (except.message) msg += ' (' + except.message + ')'; + if (wrongSoFar <= 10) { my.notify(msg,{category:'error'}); - return false; + } } } else { - my.notify('Wrong geometry value',{category:'error'}); - return false; + wrongSoFar += 1 + if (wrongSoFar <= 10) { + my.notify('Wrong geometry value',{category:'error'}); + } } return true; }); @@ -1576,7 +1726,7 @@ my.Map = Backbone.View.extend({ if (!(docs instanceof Array)) docs = [docs]; - _.each(doc,function(doc){ + _.each(docs,function(doc){ for (key in self.features._layers){ if (self.features._layers[key].cid == doc.cid){ self.features.removeLayer(self.features._layers[key]); @@ -1590,18 +1740,25 @@ my.Map = Backbone.View.extend({ // _getGeometryFromDocument: function(doc){ if (this.geomReady){ - if (this._geomFieldName){ - // We assume that the contents of the field are a valid GeoJSON object - return doc.attributes[this._geomFieldName]; - } else if (this._lonFieldName && this._latFieldName){ + if (this.state.get('geomField')){ + var value = doc.get(this.state.get('geomField')); + if (typeof(value) === 'string'){ + // We have a GeoJSON string representation + return $.parseJSON(value); + } else { + // We assume that the contents of the field are a valid GeoJSON object + return value; + } + } else if (this.state.get('lonField') && this.state.get('latField')){ // We'll create a GeoJSON like point object from the two lat/lon fields - return { - type: 'Point', - coordinates: [ - doc.attributes[this._lonFieldName], - doc.attributes[this._latFieldName] - ] - }; + var lon = doc.get(this.state.get('lonField')); + var lat = doc.get(this.state.get('latField')); + if (!isNaN(parseFloat(lon)) && !isNaN(parseFloat(lat))) { + return { + type: 'Point', + coordinates: [lon,lat] + }; + } } return null; } @@ -1613,12 +1770,16 @@ my.Map = Backbone.View.extend({ // If not found, the user can define them via the UI form. _setupGeometryField: function(){ var geomField, latField, lonField; - - this._geomFieldName = this._checkField(this.geometryFieldNames); - this._latFieldName = this._checkField(this.latitudeFieldNames); - this._lonFieldName = this._checkField(this.longitudeFieldNames); - - this.geomReady = (this._geomFieldName || (this._latFieldName && this._lonFieldName)); + this.geomReady = (this.state.get('geomField') || (this.state.get('latField') && this.state.get('lonField'))); + // should not overwrite if we have already set this (e.g. explicitly via state) + if (!this.geomReady) { + this.state.set({ + geomField: this._checkField(this.geometryFieldNames), + latField: this._checkField(this.latitudeFieldNames), + lonField: this._checkField(this.longitudeFieldNames) + }); + this.geomReady = (this.state.get('geomField') || (this.state.get('latField') && this.state.get('lonField'))); + } }, // Private: Check if a field in the current model exists in the provided @@ -1637,6 +1798,18 @@ my.Map = Backbone.View.extend({ return null; }, + // Private: Zoom to map to current features extent if any, or to the full + // extent if none. + // + _zoomToFeatures: function(){ + var bounds = this.features.getBounds(); + if (bounds){ + this.map.fitBounds(bounds); + } else { + this.map.setView(new L.LatLng(0, 0), 2); + } + }, + // Private: Sets up the Leaflet map control and the features layer. // // The map uses a base layer from [MapQuest](http://www.mapquest.com) based @@ -1661,6 +1834,24 @@ my.Map = Backbone.View.extend({ } }); + + // This will be available in the next Leaflet stable release. + // In the meantime we add it manually to our layer. + this.features.getBounds = function(){ + var bounds = new L.LatLngBounds(); + this._iterateLayers(function (layer) { + if (layer instanceof L.Marker){ + bounds.extend(layer.getLatLng()); + } else { + if (layer.getBounds){ + bounds.extend(layer.getBounds().getNorthEast()); + bounds.extend(layer.getBounds().getSouthWest()); + } + } + }, this); + return (typeof bounds.getNorthEast() !== 'undefined') ? bounds : null; + } + this.map.addLayer(this.features); this.map.setView(new L.LatLng(0, 0), 2); @@ -1895,6 +2086,85 @@ my.ColumnTransform = Backbone.View.extend({ })(jQuery, recline.View); /*jshint multistr:true */ + +// # Recline Views +// +// Recline Views are Backbone Views and in keeping with normal Backbone views +// are Widgets / Components displaying something in the DOM. Like all Backbone +// views they have a pointer to a model or a collection and is bound to an +// element. +// +// Views provided by core Recline are crudely divided into two types: +// +// * Dataset Views: a View intended for displaying a recline.Model.Dataset +// in some fashion. Examples are the Grid, Graph and Map views. +// * Widget Views: a widget used for displaying some specific (and +// smaller) aspect of a dataset or the application. Examples are +// QueryEditor and FilterEditor which both provide a way for editing (a +// part of) a `recline.Model.Query` associated to a Dataset. +// +// ## Dataset View +// +// These views are just Backbone views with a few additional conventions: +// +// 1. The model passed to the View should always be a recline.Model.Dataset instance +// 2. Views should generate their own root element rather than having it passed +// in. +// 3. Views should apply a css class named 'recline-{view-name-lower-cased} to +// the root element (and for all CSS for this view to be qualified using this +// CSS class) +// 4. Read-only mode: CSS for this view should respect/utilize +// recline-read-only class to trigger read-only behaviour (this class will +// usually be set on some parent element of the view's root element. +// 5. State: state (configuration) information for the view should be stored on +// an attribute named state that is an instance of a Backbone Model (or, more +// speficially, be an instance of `recline.Model.ObjectState`). In addition, +// a state attribute may be specified in the Hash passed to a View on +// iniitialization and this information should be used to set the initial +// state of the view. +// +// Example of state would be the set of fields being plotted in a graph +// view. +// +// More information about State can be found below. +// +// To summarize some of this, the initialize function for a Dataset View should +// look like: +// +//
    +//    initialize: {
    +//        model: {a recline.Model.Dataset instance}
    +//        // el: {do not specify - instead view should create}
    +//        state: {(optional) Object / Hash specifying initial state}
    +//        ...
    +//    }
    +// 
    +// +// Note: Dataset Views in core Recline have a common layout on disk as +// follows, where ViewName is the named of View class: +// +//
    +// src/view-{lower-case-ViewName}.js
    +// css/{lower-case-ViewName}.css
    +// test/view-{lower-case-ViewName}.js
    +// 
    +// +// ### State +// +// State information exists in order to support state serialization into the +// url or elsewhere and reloading of application from a stored state. +// +// State is available not only for individual views (as described above) but +// for the dataset (e.g. the current query). For an example of pulling together +// state from across multiple components see `recline.View.DataExplorer`. +// +// ### Writing your own Views +// +// See the existing Views. +// +// ---- + +// Standard JS module setup this.recline = this.recline || {}; this.recline.View = this.recline.View || {}; @@ -1907,47 +2177,62 @@ this.recline.View = this.recline.View || {}; // var myExplorer = new model.recline.DataExplorer({ // model: {{recline.Model.Dataset instance}} // el: {{an existing dom element}} -// views: {{page views}} -// config: {{config options -- see below}} +// views: {{dataset views}} +// state: {{state configuration -- see below}} // }); // // // ### Parameters // -// **model**: (required) Dataset instance. +// **model**: (required) recline.model.Dataset instance. // -// **el**: (required) DOM element. +// **el**: (required) DOM element to bind to. NB: the element already +// being in the DOM is important for rendering of some subviews (e.g. +// Graph). // -// **views**: (optional) the views (Grid, Graph etc) for DataExplorer to -// show. This is an array of view hashes. If not provided -// just initialize a DataGrid with id 'grid'. Example: +// **views**: (optional) the dataset views (Grid, Graph etc) for +// DataExplorer to show. This is an array of view hashes. If not provided +// initialize with (recline.View.)Grid, Graph, and Map views (with obvious id +// and labels!). // //
     // var views = [
     //   {
     //     id: 'grid', // used for routing
     //     label: 'Grid', // used for view switcher
    -//     view: new recline.View.DataGrid({
    +//     view: new recline.View.Grid({
     //       model: dataset
     //     })
     //   },
     //   {
     //     id: 'graph',
     //     label: 'Graph',
    -//     view: new recline.View.FlotGraph({
    +//     view: new recline.View.Graph({
     //       model: dataset
     //     })
     //   }
     // ];
     // 
    // -// **config**: Config options like: +// **state**: standard state config for this view. This state is slightly +// special as it includes config of many of the subviews. // -// * readOnly: true/false (default: false) value indicating whether to -// operate in read-only mode (hiding all editing options). +//
    +// state = {
    +//     query: {dataset query state - see dataset.queryState object}
    +//     view-{id1}: {view-state for this view}
    +//     view-{id2}: {view-state for }
    +//     ...
    +//     // Explorer
    +//     currentView: id of current view (defaults to first view if not specified)
    +//     readOnly: (default: false) run in read-only mode
    +// }
    +// 
    // -// NB: the element already being in the DOM is important for rendering of -// FlotGraph subview. +// Note that at present we do *not* serialize information about the actual set +// of views in use -- e.g. those specified by the views argument -- but instead +// expect either that the default views are fine or that the client to have +// initialized the DataExplorer with the relevant views themselves. my.DataExplorer = Backbone.View.extend({ template: ' \
    \ @@ -1956,7 +2241,7 @@ my.DataExplorer = Backbone.View.extend({
    \ \
    \ @@ -1979,19 +2264,14 @@ my.DataExplorer = Backbone.View.extend({
    \ ', events: { - 'click .menu-right a': 'onMenuClick' + 'click .menu-right a': '_onMenuClick', + 'click .navigation a': '_onSwitchView' }, initialize: function(options) { var self = this; this.el = $(this.el); - this.config = _.extend({ - readOnly: false - }, - options.config); - if (this.config.readOnly) { - this.setReadOnly(); - } + this._setupState(options.state); // Hash of 'page' views (i.e. those for whole page) keyed by page name if (options.views) { this.pageViews = options.views; @@ -1999,13 +2279,38 @@ my.DataExplorer = Backbone.View.extend({ this.pageViews = [{ id: 'grid', label: 'Grid', - view: new my.DataGrid({ - model: this.model - }) + view: new my.Grid({ + model: this.model, + state: this.state.get('view-grid') + }), + }, { + id: 'graph', + label: 'Graph', + view: new my.Graph({ + model: this.model, + state: this.state.get('view-graph') + }), + }, { + id: 'map', + label: 'Map', + view: new my.Map({ + model: this.model, + state: this.state.get('view-map') + }), }]; } - // this must be called after pageViews are created + // these must be called after pageViews are created this.render(); + this._bindStateChanges(); + // now do updates based on state (need to come after render) + if (this.state.get('readOnly')) { + this.setReadOnly(); + } + if (this.state.get('currentView')) { + this.updateNav(this.state.get('currentView')); + } else { + this.updateNav(this.pageViews[0].id); + } this.router = new Backbone.Router(); this.setupRouting(); @@ -2021,7 +2326,7 @@ my.DataExplorer = Backbone.View.extend({ var qs = my.parseHashQueryString(); qs.reclineQuery = JSON.stringify(self.model.queryState.toJSON()); var out = my.getNewHashForQueryString(qs); - self.router.navigate(out); + // self.router.navigate(out); }); this.model.bind('query:fail', function(error) { my.clearNotifications(); @@ -2045,11 +2350,7 @@ my.DataExplorer = Backbone.View.extend({ // note this.model and dataset returned are the same this.model.fetch() .done(function(dataset) { - var queryState = my.parseHashQueryString().reclineQuery; - if (queryState) { - queryState = JSON.parse(queryState); - } - self.model.query(queryState); + self.model.query(self.state.get('query')); }) .fail(function(error) { my.notify(error.message, {category: 'error', persist: true}); @@ -2057,12 +2358,11 @@ my.DataExplorer = Backbone.View.extend({ }, setReadOnly: function() { - this.el.addClass('read-only'); + this.el.addClass('recline-read-only'); }, render: function() { var tmplData = this.model.toTemplateJSON(); - tmplData.displayCount = this.config.displayCount; tmplData.views = this.pageViews; var template = $.mustache(this.template, tmplData); $(this.el).html(template); @@ -2089,20 +2389,22 @@ my.DataExplorer = Backbone.View.extend({ setupRouting: function() { var self = this; // Default route - this.router.route(/^(\?.*)?$/, this.pageViews[0].id, function(queryString) { - self.updateNav(self.pageViews[0].id, queryString); - }); - $.each(this.pageViews, function(idx, view) { - self.router.route(/^([^?]+)(\?.*)?/, 'view', function(viewId, queryString) { - self.updateNav(viewId, queryString); - }); +// this.router.route(/^(\?.*)?$/, this.pageViews[0].id, function(queryString) { +// self.updateNav(self.pageViews[0].id, queryString); +// }); +// $.each(this.pageViews, function(idx, view) { +// self.router.route(/^([^?]+)(\?.*)?/, 'view', function(viewId, queryString) { +// self.updateNav(viewId, queryString); +// }); +// }); + this.router.route(/.*/, 'view', function() { }); }, - updateNav: function(pageName, queryString) { + updateNav: function(pageName) { this.el.find('.navigation li').removeClass('active'); this.el.find('.navigation li a').removeClass('disabled'); - var $el = this.el.find('.navigation li a[href=#' + pageName + ']'); + var $el = this.el.find('.navigation li a[data-view="' + pageName + '"]'); $el.parent().addClass('active'); $el.addClass('disabled'); // show the specific page @@ -2117,7 +2419,7 @@ my.DataExplorer = Backbone.View.extend({ }); }, - onMenuClick: function(e) { + _onMenuClick: function(e) { e.preventDefault(); var action = $(e.target).attr('data-action'); if (action === 'filters') { @@ -2125,9 +2427,78 @@ my.DataExplorer = Backbone.View.extend({ } else if (action === 'facets') { this.$facetViewer.show(); } + }, + + _onSwitchView: function(e) { + e.preventDefault(); + var viewName = $(e.target).attr('data-view'); + this.updateNav(viewName); + this.state.set({currentView: viewName}); + }, + + // create a state object for this view and do the job of + // + // a) initializing it from both data passed in and other sources (e.g. hash url) + // + // b) ensure the state object is updated in responese to changes in subviews, query etc. + _setupState: function(initialState) { + var self = this; + // get data from the query string / hash url plus some defaults + var qs = my.parseHashQueryString(); + var query = qs.reclineQuery; + query = query ? JSON.parse(query) : self.model.queryState.toJSON(); + // backwards compatability (now named view-graph but was named graph) + var graphState = qs['view-graph'] || qs.graph; + graphState = graphState ? JSON.parse(graphState) : {}; + + // now get default data + hash url plus initial state and initial our state object with it + var stateData = _.extend({ + query: query, + 'view-graph': graphState, + backend: this.model.backend.__type__, + dataset: this.model.toJSON(), + currentView: null, + readOnly: false + }, + initialState); + this.state = new recline.Model.ObjectState(stateData); + }, + + _bindStateChanges: function() { + var self = this; + // finally ensure we update our state object when state of sub-object changes so that state is always up to date + this.model.queryState.bind('change', function() { + self.state.set({query: self.model.queryState.toJSON()}); + }); + _.each(this.pageViews, function(pageView) { + if (pageView.view.state && pageView.view.state.bind) { + var update = {}; + update['view-' + pageView.id] = pageView.view.state.toJSON(); + self.state.set(update); + pageView.view.state.bind('change', function() { + var update = {}; + update['view-' + pageView.id] = pageView.view.state.toJSON(); + // had problems where change not being triggered for e.g. grid view so let's do it explicitly + self.state.set(update, {silent: true}); + self.state.trigger('change'); + }); + } + }); } }); +// ### DataExplorer.restore +// +// Restore a DataExplorer instance from a serialized state including the associated dataset +my.DataExplorer.restore = function(state) { + var dataset = recline.Model.Dataset.restore(state); + var explorer = new my.DataExplorer({ + model: dataset, + state: state + }); + return explorer; +} + my.QueryEditor = Backbone.View.extend({ className: 'recline-query-editor', template: ' \ @@ -2403,6 +2774,9 @@ my.composeQueryString = function(queryParams) { var queryString = '?'; var items = []; $.each(queryParams, function(key, value) { + if (typeof(value) === 'object') { + value = JSON.stringify(value); + } items.push(key + '=' + value); }); queryString += items.join('&'); @@ -2484,10 +2858,27 @@ this.recline.Backend = this.recline.Backend || {}; // ## recline.Backend.Base // // Base class for backends providing a template and convenience functions. - // You do not have to inherit from this class but even when not it does provide guidance on the functions you must implement. + // You do not have to inherit from this class but even when not it does + // provide guidance on the functions you must implement. // // Note also that while this (and other Backends) are implemented as Backbone models this is just a convenience. my.Base = Backbone.Model.extend({ + // ### __type__ + // + // 'type' of this backend. This should be either the class path for this + // object as a string (e.g. recline.Backend.Memory) or for Backends within + // recline.Backend module it may be their class name. + // + // This value is used as an identifier for this backend when initializing + // backends (see recline.Model.Dataset.initialize). + __type__: 'base', + + + // ### readonly + // + // Class level attribute indicating that this backend is read-only (that + // is, cannot be written to). + readonly: true, // ### sync // @@ -2549,6 +2940,32 @@ this.recline.Backend = this.recline.Backend || {}; query: function(model, queryObj) { }, + // ### _makeRequest + // + // Just $.ajax but in any headers in the 'headers' attribute of this + // Backend instance. Example: + // + //
    +    // var jqxhr = this._makeRequest({
    +    //   url: the-url
    +    // });
    +    // 
    + _makeRequest: function(data) { + var headers = this.get('headers'); + var extras = {}; + if (headers) { + extras = { + beforeSend: function(req) { + _.each(headers, function(value, key) { + req.setRequestHeader(key, value); + }); + } + }; + } + var data = _.extend(extras, data); + return $.ajax(data); + }, + // convenience method to convert simple set of documents / rows to a QueryResult _docsToQueryResult: function(rows) { var hits = _.map(rows, function(row) { @@ -2607,6 +3024,8 @@ this.recline.Backend = this.recline.Backend || {}; // // Note that this is a **read-only** backend. my.DataProxy = my.Base.extend({ + __type__: 'dataproxy', + readonly: true, defaults: { dataproxy_url: 'http://jsonpdataproxy.appspot.com' }, @@ -2661,8 +3080,6 @@ this.recline.Backend = this.recline.Backend || {}; return dfd.promise(); } }); - recline.Model.backends['dataproxy'] = new my.DataProxy(); - }(jQuery, this.recline.Backend)); this.recline = this.recline || {}; @@ -2673,35 +3090,39 @@ this.recline.Backend = this.recline.Backend || {}; // // Connecting to [ElasticSearch](http://www.elasticsearch.org/). // - // To use this backend ensure your Dataset has one of the following - // attributes (first one found is used): + // Usage: + // + //
    +  // var backend = new recline.Backend.ElasticSearch({
    +  //   // optional as can also be provided by Dataset/Document
    +  //   url: {url to ElasticSearch endpoint i.e. ES 'type/table' url - more info below}
    +  //   // optional
    +  //   headers: {dict of headers to add to each request}
    +  // });
    +  //
    +  // @param {String} url: url for ElasticSearch type/table, e.g. for ES running
    +  // on localhost:9200 with index // twitter and type tweet it would be:
    +  // 
    +  // 
    http://localhost:9200/twitter/tweet
    + // + // This url is optional since the ES endpoint url may be specified on the the + // dataset (and on a Document by the document having a dataset attribute) by + // having one of the following (see also `_getESUrl` function): // //
       // elasticsearch_url
       // webstore_url
       // url
       // 
    - // - // This should point to the ES type url. E.G. for ES running on - // localhost:9200 with index twitter and type tweet it would be - // - //
    http://localhost:9200/twitter/tweet
    my.ElasticSearch = my.Base.extend({ - _getESUrl: function(dataset) { - var out = dataset.get('elasticsearch_url'); - if (out) return out; - out = dataset.get('webstore_url'); - if (out) return out; - out = dataset.get('url'); - return out; - }, + __type__: 'elasticsearch', + readonly: false, sync: function(method, model, options) { var self = this; if (method === "read") { if (model.__type__ == 'Dataset') { - var base = self._getESUrl(model); - var schemaUrl = base + '/_mapping'; - var jqxhr = $.ajax({ + var schemaUrl = self._getESUrl(model) + '/_mapping'; + var jqxhr = this._makeRequest({ url: schemaUrl, dataType: 'jsonp' }); @@ -2720,11 +3141,77 @@ this.recline.Backend = this.recline.Backend || {}; dfd.reject(arguments); }); return dfd.promise(); + } else if (model.__type__ == 'Document') { + var base = this._getESUrl(model.dataset) + '/' + model.id; + return this._makeRequest({ + url: base, + dataType: 'json' + }); + } + } else if (method === 'update') { + if (model.__type__ == 'Document') { + return this.upsert(model.toJSON(), this._getESUrl(model.dataset)); + } + } else if (method === 'delete') { + if (model.__type__ == 'Document') { + var url = this._getESUrl(model.dataset); + return this.delete(model.id, url); } - } else { - alert('This backend currently only supports read operations'); } }, + + // ### upsert + // + // create / update a document to ElasticSearch backend + // + // @param {Object} doc an object to insert to the index. + // @param {string} url (optional) url for ElasticSearch endpoint (if not + // defined called this._getESUrl() + upsert: function(doc, url) { + var data = JSON.stringify(doc); + url = url ? url : this._getESUrl(); + if (doc.id) { + url += '/' + doc.id; + } + return this._makeRequest({ + url: url, + type: 'POST', + data: data, + dataType: 'json' + }); + }, + + // ### delete + // + // Delete a document from the ElasticSearch backend. + // + // @param {Object} id id of object to delete + // @param {string} url (optional) url for ElasticSearch endpoint (if not + // provided called this._getESUrl() + delete: function(id, url) { + url = url ? url : this._getESUrl(); + url += '/' + id; + return this._makeRequest({ + url: url, + type: 'DELETE', + dataType: 'json' + }); + }, + + // ### _getESUrl + // + // get url to ElasticSearch endpoint (see above) + _getESUrl: function(dataset) { + if (dataset) { + var out = dataset.get('elasticsearch_url'); + if (out) return out; + out = dataset.get('webstore_url'); + if (out) return out; + out = dataset.get('url'); + return out; + } + return this.get('url'); + }, _normalizeQuery: function(queryObj) { var out = queryObj.toJSON ? queryObj.toJSON() : _.extend({}, queryObj); if (out.q !== undefined && out.q.trim() === '') { @@ -2761,7 +3248,7 @@ this.recline.Backend = this.recline.Backend || {}; var queryNormalized = this._normalizeQuery(queryObj); var data = {source: JSON.stringify(queryNormalized)}; var base = this._getESUrl(model); - var jqxhr = $.ajax({ + var jqxhr = this._makeRequest({ url: base + '/_search', data: data, dataType: 'jsonp' @@ -2782,7 +3269,6 @@ this.recline.Backend = this.recline.Backend || {}; return dfd.promise(); } }); - recline.Model.backends['elasticsearch'] = new my.ElasticSearch(); }(jQuery, this.recline.Backend)); @@ -2805,6 +3291,8 @@ this.recline.Backend = this.recline.Backend || {}; // ); //
    my.GDoc = my.Base.extend({ + __type__: 'gdoc', + readonly: true, getUrl: function(dataset) { var url = dataset.get('url'); if (url.indexOf('feeds/list') != -1) { @@ -2922,7 +3410,6 @@ this.recline.Backend = this.recline.Backend || {}; return results; } }); - recline.Model.backends['gdocs'] = new my.GDoc(); }(jQuery, this.recline.Backend)); @@ -2930,7 +3417,9 @@ this.recline = this.recline || {}; this.recline.Backend = this.recline.Backend || {}; (function($, my) { - my.loadFromCSVFile = function(file, callback) { + my.loadFromCSVFile = function(file, callback, options) { + var encoding = options.encoding || 'UTF-8'; + var metadata = { id: file.name, file: file @@ -2938,17 +3427,17 @@ this.recline.Backend = this.recline.Backend || {}; var reader = new FileReader(); // TODO reader.onload = function(e) { - var dataset = my.csvToDataset(e.target.result); + var dataset = my.csvToDataset(e.target.result, options); callback(dataset); }; reader.onerror = function (e) { alert('Failed to load file. Code: ' + e.target.error.code); }; - reader.readAsText(file); + reader.readAsText(file, encoding); }; - my.csvToDataset = function(csvString) { - var out = my.parseCSV(csvString); + my.csvToDataset = function(csvString, options) { + var out = my.parseCSV(csvString, options); fields = _.map(out[0], function(cell) { return { id: cell, label: cell }; }); @@ -2963,128 +3452,135 @@ this.recline.Backend = this.recline.Backend || {}; return dataset; }; - // Converts a Comma Separated Values string into an array of arrays. - // Each line in the CSV becomes an array. + // Converts a Comma Separated Values string into an array of arrays. + // Each line in the CSV becomes an array. // - // Empty fields are converted to nulls and non-quoted numbers are converted to integers or floats. - // - // @return The CSV parsed as an array - // @type Array - // - // @param {String} s The string to convert - // @param {Boolean} [trm=false] If set to True leading and trailing whitespace is stripped off of each non-quoted field as it is imported + // Empty fields are converted to nulls and non-quoted numbers are converted to integers or floats. // + // @return The CSV parsed as an array + // @type Array + // + // @param {String} s The string to convert + // @param {Object} options Options for loading CSV including + // @param {Boolean} [trim=false] If set to True leading and trailing whitespace is stripped off of each non-quoted field as it is imported + // @param {String} [separator=','] Separator for CSV file // Heavily based on uselesscode's JS CSV parser (MIT Licensed): // thttp://www.uselesscode.org/javascript/csv/ - my.parseCSV= function(s, trm) { - // Get rid of any trailing \n - s = chomp(s); - - var cur = '', // The character we are currently processing. - inQuote = false, - fieldQuoted = false, - field = '', // Buffer for building up the current field - row = [], - out = [], - i, - processField; - - processField = function (field) { - if (fieldQuoted !== true) { - // If field is empty set to null - if (field === '') { - field = null; - // If the field was not quoted and we are trimming fields, trim it - } else if (trm === true) { - field = trim(field); - } - - // Convert unquoted numbers to their appropriate types - if (rxIsInt.test(field)) { - field = parseInt(field, 10); - } else if (rxIsFloat.test(field)) { - field = parseFloat(field, 10); - } - } - return field; - }; + my.parseCSV= function(s, options) { + // Get rid of any trailing \n + s = chomp(s); + + var options = options || {}; + var trm = options.trim; + var separator = options.separator || ','; + var delimiter = options.delimiter || '"'; + + + var cur = '', // The character we are currently processing. + inQuote = false, + fieldQuoted = false, + field = '', // Buffer for building up the current field + row = [], + out = [], + i, + processField; + + processField = function (field) { + if (fieldQuoted !== true) { + // If field is empty set to null + if (field === '') { + field = null; + // If the field was not quoted and we are trimming fields, trim it + } else if (trm === true) { + field = trim(field); + } - for (i = 0; i < s.length; i += 1) { - cur = s.charAt(i); - - // If we are at a EOF or EOR - if (inQuote === false && (cur === ',' || cur === "\n")) { - field = processField(field); - // Add the current field to the current row - row.push(field); - // If this is EOR append row to output and flush row - if (cur === "\n") { - out.push(row); - row = []; - } - // Flush the field buffer - field = ''; - fieldQuoted = false; - } else { - // If it's not a ", add it to the field buffer - if (cur !== '"') { - field += cur; - } else { - if (!inQuote) { - // We are not in a quote, start a quote - inQuote = true; - fieldQuoted = true; - } else { - // Next char is ", this is an escaped " - if (s.charAt(i + 1) === '"') { - field += '"'; - // Skip the next char - i += 1; - } else { - // It's not escaping, so end quote - inQuote = false; - } - } - } - } - } - - // Add the last field - field = processField(field); - row.push(field); - out.push(row); - - return out; - }; - - var rxIsInt = /^\d+$/, - rxIsFloat = /^\d*\.\d+$|^\d+\.\d*$/, - // If a string has leading or trailing space, - // contains a comma double quote or a newline - // it needs to be quoted in CSV output - rxNeedsQuoting = /^\s|\s$|,|"|\n/, - trim = (function () { - // Fx 3.1 has a native trim function, it's about 10x faster, use it if it exists - if (String.prototype.trim) { - return function (s) { - return s.trim(); - }; - } else { - return function (s) { - return s.replace(/^\s*/, '').replace(/\s*$/, ''); - }; - } - }()); - - function chomp(s) { - if (s.charAt(s.length - 1) !== "\n") { - // Does not end with \n, just return string - return s; - } else { - // Remove the \n - return s.substring(0, s.length - 1); - } - } + // Convert unquoted numbers to their appropriate types + if (rxIsInt.test(field)) { + field = parseInt(field, 10); + } else if (rxIsFloat.test(field)) { + field = parseFloat(field, 10); + } + } + return field; + }; + + for (i = 0; i < s.length; i += 1) { + cur = s.charAt(i); + + // If we are at a EOF or EOR + if (inQuote === false && (cur === separator || cur === "\n")) { + field = processField(field); + // Add the current field to the current row + row.push(field); + // If this is EOR append row to output and flush row + if (cur === "\n") { + out.push(row); + row = []; + } + // Flush the field buffer + field = ''; + fieldQuoted = false; + } else { + // If it's not a delimiter, add it to the field buffer + if (cur !== delimiter) { + field += cur; + } else { + if (!inQuote) { + // We are not in a quote, start a quote + inQuote = true; + fieldQuoted = true; + } else { + // Next char is delimiter, this is an escaped delimiter + if (s.charAt(i + 1) === delimiter) { + field += delimiter; + // Skip the next char + i += 1; + } else { + // It's not escaping, so end quote + inQuote = false; + } + } + } + } + } + + // Add the last field + field = processField(field); + row.push(field); + out.push(row); + + return out; + }; + + var rxIsInt = /^\d+$/, + rxIsFloat = /^\d*\.\d+$|^\d+\.\d*$/, + // If a string has leading or trailing space, + // contains a comma double quote or a newline + // it needs to be quoted in CSV output + rxNeedsQuoting = /^\s|\s$|,|"|\n/, + trim = (function () { + // Fx 3.1 has a native trim function, it's about 10x faster, use it if it exists + if (String.prototype.trim) { + return function (s) { + return s.trim(); + }; + } else { + return function (s) { + return s.replace(/^\s*/, '').replace(/\s*$/, ''); + }; + } + }()); + + function chomp(s) { + if (s.charAt(s.length - 1) !== "\n") { + // Does not end with \n, just return string + return s; + } else { + // Remove the \n + return s.substring(0, s.length - 1); + } + } }(jQuery, this.recline.Backend)); @@ -3110,7 +3606,7 @@ this.recline.Backend = this.recline.Backend || {}; if (!metadata.id) { metadata.id = String(Math.floor(Math.random() * 100000000) + 1); } - var backend = recline.Model.backends['memory']; + var backend = new recline.Backend.Memory(); var datasetInfo = { documents: data, metadata: metadata @@ -3125,7 +3621,7 @@ this.recline.Backend = this.recline.Backend || {}; } } backend.addDataset(datasetInfo); - var dataset = new recline.Model.Dataset({id: metadata.id}, 'memory'); + var dataset = new recline.Model.Dataset({id: metadata.id}, backend); dataset.fetch(); return dataset; }; @@ -3160,6 +3656,8 @@ this.recline.Backend = this.recline.Backend || {}; // etc ... // my.Memory = my.Base.extend({ + __type__: 'memory', + readonly: false, initialize: function() { this.datasets = {}; }, @@ -3207,13 +3705,9 @@ this.recline.Backend = this.recline.Backend || {}; var out = {}; var numRows = queryObj.size; var start = queryObj.from; - results = this.datasets[model.id].documents; - _.each(queryObj.filters, function(filter) { - results = _.filter(results, function(doc) { - var fieldId = _.keys(filter.term)[0]; - return (doc[fieldId] == filter.term[fieldId]); - }); - }); + var results = this.datasets[model.id].documents; + results = this._applyFilters(results, queryObj); + results = this._applyFreeTextQuery(model, results, queryObj); // not complete sorting! _.each(queryObj.sort, function(sortObj) { var fieldName = _.keys(sortObj)[0]; @@ -3231,6 +3725,43 @@ this.recline.Backend = this.recline.Backend || {}; return dfd.promise(); }, + // in place filtering + _applyFilters: function(results, queryObj) { + _.each(queryObj.filters, function(filter) { + results = _.filter(results, function(doc) { + var fieldId = _.keys(filter.term)[0]; + return (doc[fieldId] == filter.term[fieldId]); + }); + }); + return results; + }, + + // we OR across fields but AND across terms in query string + _applyFreeTextQuery: function(dataset, results, queryObj) { + if (queryObj.q) { + var terms = queryObj.q.split(' '); + results = _.filter(results, function(rawdoc) { + var matches = true; + _.each(terms, function(term) { + var foundmatch = false; + dataset.fields.each(function(field) { + var value = rawdoc[field.id]; + if (value !== null) { value = value.toString(); } + // TODO regexes? + foundmatch = foundmatch || (value === term); + // TODO: early out (once we are true should break to spare unnecessary testing) + // if (foundmatch) return true; + }); + matches = matches && foundmatch; + // TODO: early out (once false should break to spare unnecessary testing) + // if (!matches) return false; + }); + return matches; + }); + } + return results; + }, + _computeFacets: function(documents, queryObj) { var facetResults = {}; if (!queryObj.facets) { @@ -3267,6 +3798,5 @@ this.recline.Backend = this.recline.Backend || {}; return facetResults; } }); - recline.Model.backends['memory'] = new my.Memory(); }(jQuery, this.recline.Backend)); diff --git a/ckan/templates/_snippet/add-related.html b/ckan/templates/_snippet/add-related.html new file mode 100644 index 00000000000..e5b2a64a1a3 --- /dev/null +++ b/ckan/templates/_snippet/add-related.html @@ -0,0 +1,48 @@ + + + diff --git a/ckan/templates/_snippet/data-api-help.html b/ckan/templates/_snippet/data-api-help.html index 5a499c938b8..a3874172ccf 100644 --- a/ckan/templates/_snippet/data-api-help.html +++ b/ckan/templates/_snippet/data-api-help.html @@ -19,7 +19,7 @@

    support. Further information in the main CKAN Data API and DataStore documentation.

    - +
    Endpoints » @@ -35,6 +35,16 @@

    + + + + + + + + - - - -
    Base${datastore_api}
    Query + ${datastore_api}/_search +
    Query example @@ -47,10 +57,6 @@

    ${datastore_api}/_mapping?pretty=true

    Base${datastore_api}

    @@ -62,55 +68,37 @@

    Querying »

    -
    +
    -

    Basic queries can be done using the q - parameter in the query string which supports the Lucene - query parser syntax and hence filters on specific fields - (e.g. fieldname:value), wildcards (e.g. abc*) - and more. Full query parameters and options in the ElasticSearch - URI request docs.

    - -

    More complex queries, including those that involve - faceting and statistical operations, should use the full ElasticSearch - query language in which the query is a JSON structure sent in the - ?source= parameter. See ElasticSearch - query documentation.

    + Query example (first 5 results) +

    + ${datastore_api}/_search?size=5&pretty=true +

    -

    JSONP support is available via a simple callback query parameter: - ?callback=my_callback_name.

    -
    -
    -
    + Query example (results with 'jones' in title field) +

    + ${datastore_api}/_search?q=title:jones&size=5&pretty=true +

    -
    - -
    -
    -

    The following examples utilize the cURL - command line utility. If you prefer, you you can just open the relevant urls in - your browser.

    + Schema (Mapping) +

    + ${datastore_api}/_mapping?pretty=true +

    -
    -// added pretty=true to get the json results pretty printed
    -curl ${datastore_api}/_search?q=title:jones&size=5&pretty=true
    + Endpoint (for clients) +

    + ${datastore_api} +

    +
    -
    +

    A simple ajax (JSONP) request to the data API using jQuery.

    @@ -119,7 +107,8 @@ 

    q: 'title:jones' // query on the title field for 'jones' }; $.ajax({ - url: ${datastore_api}/_search, + url: '${datastore_api}/_search', + data: data, dataType: 'jsonp', success: function(data) { alert('Total results found: ' + data.hits.total) @@ -128,6 +117,22 @@

    + +
    + +
    +
    +
    +import urllib
    +url = '${datastore_api}/_search?size=5&q=title:jones'
    +fileobj = urllib.urlopen(url)
    +print fileobj.read()
    +
    +
    +
    +

    diff --git a/ckan/templates/_util.html b/ckan/templates/_util.html index f470b072113..8e320d8519d 100644 --- a/ckan/templates/_util.html +++ b/ckan/templates/_util.html @@ -40,19 +40,11 @@
    • - ${h.link_to(tag['name'], h.url_for(controller='tag', action='read', id=tag['name']))} + ${h.link_to(tag['display_name'], h.url_for(controller='tag', action='read', id=tag['name']))}
    - -
      -
    • - ${h.link_to(tag, h.url_for(controller='tag', action='read', id=tag))} -
    • -
    - - ${facet_sidebar('tags')} - ${facet_sidebar('res_format')} + ${facet_div('tags', 'Tags')} + ${facet_div('res_format', 'Resource Formats')} @@ -42,48 +43,12 @@

    State: ${c.group['state']}

    Datasets

    - - + ${field_list()}

    You searched for "${c.q}". ${c.page.item_count} datasets found.

    ${c.page.pager()} - - -
    - - ${package.get('title') or package.get('name')} -    - - - - ${resource.get('format')} - - - -

    ${h.markdown_extract(package.notes)}

    - - - - - [Open Data] - - - - ${h.icon('lock')} Not Openly Licensed - - -
    -
    + ${package_list_from_dict(c.page.items)} ${c.page.pager()}
    diff --git a/ckan/templates/js_strings.html b/ckan/templates/js_strings.html index 5ba337c9d2a..76a0d7e580f 100644 --- a/ckan/templates/js_strings.html +++ b/ckan/templates/js_strings.html @@ -51,6 +51,7 @@ CKAN.Strings.datastoreEnabled = "${_('DataStore enabled')}"; CKAN.Strings.sizeBracketsBytes = "${_('Size (Bytes)')}"; CKAN.Strings.mimetype = "${_('Mimetype')}"; + CKAN.Strings.created = "${_('Created')}"; CKAN.Strings.lastModified = "${_('Last Modified')}"; CKAN.Strings.mimetypeInner = "${_('Mimetype (Inner)')}"; CKAN.Strings.hash = "${_('Hash')}"; diff --git a/ckan/templates/layout_base.html b/ckan/templates/layout_base.html index 1518d7229a6..a983d4bc606 100644 --- a/ckan/templates/layout_base.html +++ b/ckan/templates/layout_base.html @@ -266,6 +266,28 @@

    Meta

    }); + + ${optional_footer()} diff --git a/ckan/templates/package/layout.html b/ckan/templates/package/layout.html index 2751557fd0e..5015093bed3 100644 --- a/ckan/templates/package/layout.html +++ b/ckan/templates/package/layout.html @@ -21,13 +21,13 @@
    @@ -35,18 +35,23 @@
  • ${h.subnav_link(h.icon('page_stack') + _('History'), controller='package', action='history', id=c.pkg.name)}
  • + +
  • ${h.subnav_link(h.icon('package') + _('Related') + ' (%s)' % c.related_count, controller='related', action='list', id=c.pkg.name)}
  • +
  • |
  • ${h.subnav_link(h.icon('package_edit') + _('Settings'), controller='package', action='edit', id=c.pkg.name)}
  • + +
  • ${h.subnav_link(h.icon('lock') + _('Authorization'), controller='package', action='authz', id=c.pkg.name)}
  • - + diff --git a/ckan/templates/package/related_list.html b/ckan/templates/package/related_list.html new file mode 100644 index 00000000000..49a75ea5a33 --- /dev/null +++ b/ckan/templates/package/related_list.html @@ -0,0 +1,56 @@ + + + no-sidebar + + + ${c.pkg_dict.get('title', c.pkg_dict['name'])} + - Related + + + ${c.pkg_dict['title']} - Related + + +
    + ${add_related(c.pkg)} +

    Related items Add related item

    +
    + + + +
    + + +
    + + + + + + + + + + + + + + + diff --git a/ckan/templates/package/resource_embedded_dataviewer.html b/ckan/templates/package/resource_embedded_dataviewer.html new file mode 100644 index 00000000000..5022a4dec44 --- /dev/null +++ b/ckan/templates/package/resource_embedded_dataviewer.html @@ -0,0 +1,98 @@ + + + + + + + + + + + + + + + + ${h.dataset_display_name(c.package)} / + ${h.resource_display_name(c.resource)} - Dataset - Resource + + +
    +
    +
    +
    + +
    + + + + + + + + + + + + diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index b6c30e640c6..619c6368cf6 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -12,6 +12,7 @@ py:strip=""> + @@ -20,8 +21,9 @@ - + + @@ -104,6 +120,8 @@ ${data_api_help(c.datastore_api)} + ${data_viewer_embed_dialog()} +
    Last updated
    @@ -151,7 +169,11 @@
    -

    Preview

    +
    +

    Preview

    + + +
    diff --git a/ckan/templates/package/search.html b/ckan/templates/package/search.html index a73bfa2d616..cee817e4571 100644 --- a/ckan/templates/package/search.html +++ b/ckan/templates/package/search.html @@ -20,10 +20,10 @@

    Add a dataset

    Register it now.

    - - ${facet_sidebar('tags')} - ${facet_sidebar('res_format')} - ${facet_sidebar('groups', label=h.group_name_to_title)} + + ${facet_div('tags', 'Tags')} + ${facet_div('res_format', 'Resource Formats')} + ${facet_div('groups', 'Groups')}
  • Other access

    @@ -46,7 +46,6 @@

    Other access

    ${field_list()} -

    There was an error while searching. Please try again.

    diff --git a/ckan/templates/package/search_form.html b/ckan/templates/package/search_form.html index 5313687e5ae..ba15882d89d 100644 --- a/ckan/templates/package/search_form.html +++ b/ckan/templates/package/search_form.html @@ -5,7 +5,7 @@ py:strip="" > -