From a62293aff9884d9a4b3a5f6e323fffdd7c1d6a10 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 24 Jul 2012 16:29:56 +0100 Subject: [PATCH 001/130] Add basic datastore extension skeleton with create logic/auth functions and tests. --- ckanext/datastore/__init__.py | 0 ckanext/datastore/db.py | 25 +++++++++ ckanext/datastore/logic/__init__.py | 0 ckanext/datastore/logic/action/__init__.py | 0 ckanext/datastore/logic/action/create.py | 43 ++++++++++++++++ ckanext/datastore/logic/auth/__init__.py | 0 ckanext/datastore/logic/auth/create.py | 12 +++++ ckanext/datastore/logic/schema.py | 22 ++++++++ ckanext/datastore/plugin.py | 17 +++++++ ckanext/datastore/tests/__init__.py | 0 ckanext/datastore/tests/test_datastore.py | 59 ++++++++++++++++++++++ setup.py | 1 + 12 files changed, 179 insertions(+) create mode 100644 ckanext/datastore/__init__.py create mode 100644 ckanext/datastore/db.py create mode 100644 ckanext/datastore/logic/__init__.py create mode 100644 ckanext/datastore/logic/action/__init__.py create mode 100644 ckanext/datastore/logic/action/create.py create mode 100644 ckanext/datastore/logic/auth/__init__.py create mode 100644 ckanext/datastore/logic/auth/create.py create mode 100644 ckanext/datastore/logic/schema.py create mode 100644 ckanext/datastore/plugin.py create mode 100644 ckanext/datastore/tests/__init__.py create mode 100644 ckanext/datastore/tests/test_datastore.py diff --git a/ckanext/datastore/__init__.py b/ckanext/datastore/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py new file mode 100644 index 00000000000..19845d05361 --- /dev/null +++ b/ckanext/datastore/db.py @@ -0,0 +1,25 @@ + + +def create(resource_id, fields, rows): + ''' + The first row will be used to guess types not in the fields and the + guessed types will be added to the headers permanently. + Consecutive rows have to conform to the field definitions. + + rows can be empty so that you can just set the fields. + + fields are optional but needed if you want to do type hinting or + add extra information for certain columns or to explicitly + define ordering. + + eg [{"id": "dob", "label": ""Date of Birth", + "type": "timestamp" ,"concept": "day"}, + {"name": "some_stuff": ..]. + + A header items values can not be changed after it has been defined + nor can the ordering of them be changed. They can be extended though. + + Any error results in total failure! For now pass back the actual error. + Should be transactional. + ''' + pass diff --git a/ckanext/datastore/logic/__init__.py b/ckanext/datastore/logic/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/datastore/logic/action/__init__.py b/ckanext/datastore/logic/action/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/datastore/logic/action/create.py b/ckanext/datastore/logic/action/create.py new file mode 100644 index 00000000000..c7e02d70237 --- /dev/null +++ b/ckanext/datastore/logic/action/create.py @@ -0,0 +1,43 @@ +import logging +import ckan.logic as logic +import ckan.logic.action +import ckan.lib.dictization +import ckan.plugins as p +import ckanext.datastore.logic.schema +import ckanext.datastore.db as db + +log = logging.getLogger(__name__) + +_validate = ckan.lib.navl.dictization_functions.validate +_check_access = logic.check_access +_get_or_bust = logic.get_or_bust + + +def datastore_create(context, data_dict): + '''Adds a new table to the datastore. + + :param resource_id: resource id that the data is going to be stored under. + :type resource_id: string + :param fields: fields/columns and their extra metadata. + :type fields: list of dictionaries + :param records: the data, eg: [{"dob": "2005", "some_stuff": ['a', b']}] + :type records: list of dictionaries + + :returns: the newly created data object. + :rtype: dictionary + + ''' + model = _get_or_bust(context, 'model') + resource_id = _get_or_bust(data_dict, 'resource_id') + fields = data_dict.get('fields') + records = _get_or_bust(data_dict, 'records') + + _check_access('datastore_create', context, data_dict) + + schema = ckanext.datastore.logic.schema.default_datastore_create_schema() + data, errors = _validate(data_dict, schema, context) + if errors: + model.Session.rollback() + raise p.toolkit.ValidationError(errors) + + return db.create(resource_id, fields, records) diff --git a/ckanext/datastore/logic/auth/__init__.py b/ckanext/datastore/logic/auth/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/datastore/logic/auth/create.py b/ckanext/datastore/logic/auth/create.py new file mode 100644 index 00000000000..d5db8d0ebdf --- /dev/null +++ b/ckanext/datastore/logic/auth/create.py @@ -0,0 +1,12 @@ +import ckan.plugins as p + + +def datastore_create(context, data_dict): + model = context['model'] + user = context['user'] + userobj = model.User.get(user) + + if userobj: + return {'success': True} + return {'success': False, + 'msg': p.toolkit._('You must be logged in to use the datastore.')} diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py new file mode 100644 index 00000000000..b8aaa29b6df --- /dev/null +++ b/ckanext/datastore/logic/schema.py @@ -0,0 +1,22 @@ +from ckan.lib.navl.validators import (not_empty, + not_missing, + empty, + ignore_missing, + ignore) + + +def default_fields_schema(): + return { + 'id': [not_missing, not_empty, unicode], + 'type': [not_missing, not_empty, unicode], + 'label': [ignore_missing, unicode], + } + + +def default_datastore_create_schema(): + return { + 'resource_id': [not_missing, not_empty, unicode], + 'fields': default_fields_schema(), + 'records': [ignore], + '__extras': [empty], + } diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py new file mode 100644 index 00000000000..2cb6730d714 --- /dev/null +++ b/ckanext/datastore/plugin.py @@ -0,0 +1,17 @@ +import ckan.plugins as p +import ckanext.datastore.logic.action.create as action_create +import ckanext.datastore.logic.auth.create as auth_create + + +class DatastorePlugin(p.SingletonPlugin): + ''' + Datastore plugin. + ''' + p.implements(p.IActions) + p.implements(p.IAuthFunctions) + + def get_actions(self): + return {'datastore_create': action_create.datastore_create} + + def get_auth_functions(self): + return {'datastore_create': auth_create.datastore_create} diff --git a/ckanext/datastore/tests/__init__.py b/ckanext/datastore/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py new file mode 100644 index 00000000000..71dad80b1b9 --- /dev/null +++ b/ckanext/datastore/tests/test_datastore.py @@ -0,0 +1,59 @@ +import json +import ckan.plugins as p +import ckan.lib.create_test_data as ctd +import ckan.model as model +import ckan.tests as tests + + +class TestDatastore(tests.WsgiAppCase): + sysadmin_user = None + normal_user = None + + @classmethod + def setup_class(cls): + p.load('datastore') + ctd.CreateTestData.create() + cls.sysadmin_user = model.User.get('testsysadmin') + cls.normal_user = model.User.get('annafan') + + @classmethod + def teardown_class(cls): + model.repo.rebuild_db() + + def test_create_empty_fails(self): + postparams = '%s=1' % json.dumps({}) + res = self.app.post('/api/action/datastore_create', params=postparams, + status=409) + res_dict = json.loads(res.body) + assert res_dict['success'] is False + + def test_create_requires_auth(self): + resource = model.Package.get('annakarenina').resources[0] + data = { + 'resource_id': resource.id + } + postparams = '%s=1' % json.dumps(data) + res = self.app.post('/api/action/datastore_create', params=postparams, + status=403) + res_dict = json.loads(res.body) + assert res_dict['success'] is False + + def test_create_basic(self): + resource = model.Package.get('annakarenina').resources[0] + data = { + 'resource_id': resource.id, + 'fields': [{'id': 'book', 'label': 'Name', 'type': 'text'}, + {'id': 'author', 'label': 'Author ', 'type': 'text'}], + 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, + {'book': 'warandpeace', 'author': 'tolstoy'}] + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + + assert res_dict['success'] is True + assert res_dict['result']['resource_id'] == data['resource_id'] + assert res_dict['result']['fields'] == data['fields'] + assert res_dict['result']['records'] == data['records'] diff --git a/setup.py b/setup.py index 74187be3b0c..02ee9ea1142 100644 --- a/setup.py +++ b/setup.py @@ -107,6 +107,7 @@ multilingual_tag=ckanext.multilingual.plugin:MultilingualTag organizations=ckanext.organizations.forms:OrganizationForm organizations_dataset=ckanext.organizations.forms:OrganizationDatasetForm + datastore=ckanext.datastore.plugin:DatastorePlugin test_tag_vocab_plugin=ckanext.test_tag_vocab_plugin:MockVocabTagsPlugin [ckan.system_plugins] From 1a8709debf5d6b4e89162cc8a38c912585d2d376 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 24 Jul 2012 17:12:26 +0100 Subject: [PATCH 002/130] Bug fix: records field can be empty. --- ckanext/datastore/logic/action/create.py | 2 +- ckanext/datastore/logic/schema.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/datastore/logic/action/create.py b/ckanext/datastore/logic/action/create.py index c7e02d70237..da6491b0e63 100644 --- a/ckanext/datastore/logic/action/create.py +++ b/ckanext/datastore/logic/action/create.py @@ -30,7 +30,7 @@ def datastore_create(context, data_dict): model = _get_or_bust(context, 'model') resource_id = _get_or_bust(data_dict, 'resource_id') fields = data_dict.get('fields') - records = _get_or_bust(data_dict, 'records') + records = data_dict.get('records') _check_access('datastore_create', context, data_dict) diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index b8aaa29b6df..595d753d169 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -14,6 +14,7 @@ def default_fields_schema(): def default_datastore_create_schema(): + # TODO: resource_id should have a resource_id_exists validator return { 'resource_id': [not_missing, not_empty, unicode], 'fields': default_fields_schema(), From ee0cb2dae07281430b784880284999b9c806f7e7 Mon Sep 17 00:00:00 2001 From: kindly Date: Wed, 25 Jul 2012 00:38:24 +0100 Subject: [PATCH 003/130] [2733] add engines and basic transaction scaffolding for create --- ckanext/datastore/db.py | 76 +++++++++++++++++++++++- ckanext/datastore/logic/action/create.py | 17 +++--- 2 files changed, 84 insertions(+), 9 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 19845d05361..4951987e660 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1,6 +1,57 @@ +import sqlalchemy +from pylons import config +_pg_types = {} +_type_names = set() +_engines = {} -def create(resource_id, fields, rows): +def _get_engine(context, data_dict): + ''' Get either read or write engine''' + connection_type = data_dict.get('connection_type', 'write') + engine = _engines.get(connection_type) + + if not engine: + config_option = 'ckan.datastore_{}_url'.format(connection_type) + url = config.get(config_option) + assert url, 'Config option ' + config_option + ' not defined' + engine = sqlalchemy.create_engine(url) + _engines[connection_type] = engine + return engine + +def _cache_types(context, data_dict=None): + if not _pg_types: + connection = context['connection'] + results = connection.execute( + 'select oid, typname from pg_type;' + ) + for result in results: + _pg_types[result[0]] = result[1] + _type_names.add(result[1]) + +def _get_type(context, oid): + _cache_types(context) + return _pg_types[oid] + +def check_fields(context, fields): + _cache_types(context) + ## check if fieds are in in _type_names + pass + +def create_table(context, data_dict): + '''create table from combination of fields and first row of data''' + check_fields(context, data_dict.get('fields')) + pass + +def alter_table(context, data_dict): + '''alter table from combination of fields and first row of data''' + check_fields(context, data_dict.get('fields')) + pass + +def insert_data(context, data_dict): + '''insert all data from records''' + pass + +def create(context, data_dict): ''' The first row will be used to guess types not in the fields and the guessed types will be added to the headers permanently. @@ -22,4 +73,25 @@ def create(resource_id, fields, rows): Any error results in total failure! For now pass back the actual error. Should be transactional. ''' - pass + + engine = _get_engine(context, {'connection_type': 'write'}) + context['connection'] = engine.connect() + ## close connection at all cost. + try: + ## check if table already existes + trans = context['connection'].begin() + result = context['connection'].execute( + 'select * from pg_tables where tablename = %s', + data_dict['resource_id'] + ).fetchone() + if not result: + create_table(context, data_dict) + else: + alter_table(context, data_dict) + insert_data(context, data_dict) + except: + trans.rollback() + raise + finally: + context['connection'].close() + diff --git a/ckanext/datastore/logic/action/create.py b/ckanext/datastore/logic/action/create.py index c7e02d70237..ca0321bf8fb 100644 --- a/ckanext/datastore/logic/action/create.py +++ b/ckanext/datastore/logic/action/create.py @@ -30,14 +30,17 @@ def datastore_create(context, data_dict): model = _get_or_bust(context, 'model') resource_id = _get_or_bust(data_dict, 'resource_id') fields = data_dict.get('fields') - records = _get_or_bust(data_dict, 'records') + records = data_dict.get('records') _check_access('datastore_create', context, data_dict) - schema = ckanext.datastore.logic.schema.default_datastore_create_schema() - data, errors = _validate(data_dict, schema, context) - if errors: - model.Session.rollback() - raise p.toolkit.ValidationError(errors) + # Not sure need schema as will be too dificulut to make + # as records could be deeply nested.. - return db.create(resource_id, fields, records) + #schema = ckanext.datastore.logic.schema.default_datastore_create_schema() + #data, errors = _validate(data_dict, schema, context) + #if errors: + # model.Session.rollback() + # raise p.toolkit.ValidationError(errors) + + return db.create(context, data_dict) From 9bb2462a8a913e8d0ea75234113bb04f468e70c9 Mon Sep 17 00:00:00 2001 From: John Glover Date: Wed, 25 Jul 2012 10:34:31 +0100 Subject: [PATCH 004/130] Check for datastore_write_url at startup, pass to db from logic functions. Don't validate fields and records. --- ckanext/datastore/db.py | 35 ++++++++++++++---------- ckanext/datastore/logic/action/create.py | 21 +++++++------- ckanext/datastore/logic/schema.py | 11 +------- ckanext/datastore/plugin.py | 11 ++++++++ 4 files changed, 44 insertions(+), 34 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 4951987e660..dccd1c2cfe8 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1,23 +1,25 @@ import sqlalchemy -from pylons import config _pg_types = {} _type_names = set() _engines = {} + +class InvalidType(Exception): + pass + + def _get_engine(context, data_dict): - ''' Get either read or write engine''' - connection_type = data_dict.get('connection_type', 'write') - engine = _engines.get(connection_type) + '''Get either read or write engine''' + connection_url = data_dict['connection_url'] + engine = _engines.get(connection_url) if not engine: - config_option = 'ckan.datastore_{}_url'.format(connection_type) - url = config.get(config_option) - assert url, 'Config option ' + config_option + ' not defined' - engine = sqlalchemy.create_engine(url) - _engines[connection_type] = engine + engine = sqlalchemy.create_engine(connection_url) + _engines[connection_url] = engine return engine + def _cache_types(context, data_dict=None): if not _pg_types: connection = context['connection'] @@ -28,29 +30,35 @@ def _cache_types(context, data_dict=None): _pg_types[result[0]] = result[1] _type_names.add(result[1]) + def _get_type(context, oid): _cache_types(context) return _pg_types[oid] + def check_fields(context, fields): _cache_types(context) ## check if fieds are in in _type_names pass + def create_table(context, data_dict): '''create table from combination of fields and first row of data''' check_fields(context, data_dict.get('fields')) pass + def alter_table(context, data_dict): '''alter table from combination of fields and first row of data''' check_fields(context, data_dict.get('fields')) pass + def insert_data(context, data_dict): '''insert all data from records''' pass + def create(context, data_dict): ''' The first row will be used to guess types not in the fields and the @@ -73,12 +81,12 @@ def create(context, data_dict): Any error results in total failure! For now pass back the actual error. Should be transactional. ''' - - engine = _get_engine(context, {'connection_type': 'write'}) + engine = _get_engine(context, data_dict) context['connection'] = engine.connect() - ## close connection at all cost. + + # close connection at all cost. try: - ## check if table already existes + # check if table already existes trans = context['connection'].begin() result = context['connection'].execute( 'select * from pg_tables where tablename = %s', @@ -94,4 +102,3 @@ def create(context, data_dict): raise finally: context['connection'].close() - diff --git a/ckanext/datastore/logic/action/create.py b/ckanext/datastore/logic/action/create.py index ca0321bf8fb..879ec7d32e6 100644 --- a/ckanext/datastore/logic/action/create.py +++ b/ckanext/datastore/logic/action/create.py @@ -1,4 +1,5 @@ import logging +import pylons import ckan.logic as logic import ckan.logic.action import ckan.lib.dictization @@ -28,19 +29,19 @@ def datastore_create(context, data_dict): ''' model = _get_or_bust(context, 'model') - resource_id = _get_or_bust(data_dict, 'resource_id') - fields = data_dict.get('fields') - records = data_dict.get('records') _check_access('datastore_create', context, data_dict) - # Not sure need schema as will be too dificulut to make - # as records could be deeply nested.. + # TODO: remove this check for resource ID when the resource_id_exists + # validator has been created. + _get_or_bust(data_dict, 'resource_id') - #schema = ckanext.datastore.logic.schema.default_datastore_create_schema() - #data, errors = _validate(data_dict, schema, context) - #if errors: - # model.Session.rollback() - # raise p.toolkit.ValidationError(errors) + schema = ckanext.datastore.logic.schema.default_datastore_create_schema() + data, errors = _validate(data_dict, schema, context) + if errors: + model.Session.rollback() + raise p.toolkit.ValidationError(errors) + + data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] return db.create(context, data_dict) diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index 595d753d169..7c1f170e95e 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -1,23 +1,14 @@ from ckan.lib.navl.validators import (not_empty, not_missing, empty, - ignore_missing, ignore) -def default_fields_schema(): - return { - 'id': [not_missing, not_empty, unicode], - 'type': [not_missing, not_empty, unicode], - 'label': [ignore_missing, unicode], - } - - def default_datastore_create_schema(): # TODO: resource_id should have a resource_id_exists validator return { 'resource_id': [not_missing, not_empty, unicode], - 'fields': default_fields_schema(), + 'fields': [ignore], 'records': [ignore], '__extras': [empty], } diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 2cb6730d714..ab2fee05fb2 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -3,13 +3,24 @@ import ckanext.datastore.logic.auth.create as auth_create +class DatastoreException(Exception): + pass + + class DatastorePlugin(p.SingletonPlugin): ''' Datastore plugin. ''' + p.implements(p.IConfigurable, inherit=True) p.implements(p.IActions) p.implements(p.IAuthFunctions) + def configure(self, config): + # check for ckan.datastore_write_url + if (not 'ckan.datastore_write_url' in config): + error_msg = 'ckan.datastore_write_url not found in config' + raise DatastoreException(error_msg) + def get_actions(self): return {'datastore_create': action_create.datastore_create} From cce90af7663ab747426ad9165879ebd6c2c025da Mon Sep 17 00:00:00 2001 From: John Glover Date: Wed, 25 Jul 2012 11:30:46 +0100 Subject: [PATCH 005/130] Check field types, raise an error if any are invalid. --- ckanext/datastore/db.py | 10 ++++++---- ckanext/datastore/logic/action/create.py | 5 ++++- ckanext/datastore/tests/test_datastore.py | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index dccd1c2cfe8..7b4850edea0 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -10,7 +10,7 @@ class InvalidType(Exception): def _get_engine(context, data_dict): - '''Get either read or write engine''' + 'Get either read or write engine.' connection_url = data_dict['connection_url'] engine = _engines.get(connection_url) @@ -20,7 +20,7 @@ def _get_engine(context, data_dict): return engine -def _cache_types(context, data_dict=None): +def _cache_types(context): if not _pg_types: connection = context['connection'] results = connection.execute( @@ -37,9 +37,11 @@ def _get_type(context, oid): def check_fields(context, fields): + 'Check if field types are valid.' _cache_types(context) - ## check if fieds are in in _type_names - pass + for field in fields: + if not field['type'] in _type_names: + raise InvalidType('%s is not a valid type' % field['type']) def create_table(context, data_dict): diff --git a/ckanext/datastore/logic/action/create.py b/ckanext/datastore/logic/action/create.py index 879ec7d32e6..f2a59d144b5 100644 --- a/ckanext/datastore/logic/action/create.py +++ b/ckanext/datastore/logic/action/create.py @@ -44,4 +44,7 @@ def datastore_create(context, data_dict): data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] - return db.create(context, data_dict) + try: + return db.create(context, data_dict) + except db.InvalidType as e: + raise p.toolkit.ValidationError({'fields': e.message}) diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index 71dad80b1b9..efa98e3788a 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -38,6 +38,20 @@ def test_create_requires_auth(self): res_dict = json.loads(res.body) assert res_dict['success'] is False + def test_create_invalid_field(self): + resource = model.Package.get('annakarenina').resources[0] + data = { + 'resource_id': resource.id, + 'fields': [{'id': 'book', 'label': 'Name', 'type': 'INVALID'}, + {'id': 'author', 'label': 'Author ', 'type': 'INVALID'}] + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth, status=409) + res_dict = json.loads(res.body) + assert res_dict['success'] is False + def test_create_basic(self): resource = model.Package.get('annakarenina').resources[0] data = { From c1926898a885a1e46e7e3931275228dc49cd471c Mon Sep 17 00:00:00 2001 From: John Glover Date: Wed, 25 Jul 2012 14:59:42 +0100 Subject: [PATCH 006/130] [#2733] Add basic create table functionality. --- ckanext/datastore/db.py | 72 +++++++++++++++++++++-- ckanext/datastore/logic/action/create.py | 5 +- ckanext/datastore/tests/test_datastore.py | 44 +++++++++++--- 3 files changed, 102 insertions(+), 19 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 7b4850edea0..e388bd3694e 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1,12 +1,20 @@ import sqlalchemy +import ckan.plugins as p _pg_types = {} _type_names = set() _engines = {} -class InvalidType(Exception): - pass +def _is_valid_field_name(name): + ''' + Check that field name is valid: + * can't start with underscore + * can't contain double quote (") + ''' + if name.startswith('_') or '"' in name: + return False + return True def _get_engine(context, data_dict): @@ -36,24 +44,75 @@ def _get_type(context, oid): return _pg_types[oid] +def _guess_type(field): + 'Simple guess type of field, only allowed are integer, numeric and text' + data_types = set([int, float]) + + for data_type in list(data_types): + try: + data_type(field) + except (TypeError, ValueError): + data_types.discard(data_type) + if not data_types: + break + + if int in data_types: + return 'integer' + elif float in data_types: + return 'numeric' + else: + return 'text' + + def check_fields(context, fields): 'Check if field types are valid.' _cache_types(context) for field in fields: if not field['type'] in _type_names: - raise InvalidType('%s is not a valid type' % field['type']) + raise p.toolkit.ValidationError({ + 'fields': '{0} is not a valid field type'.format(field['type']) + }) + elif not _is_valid_field_name(field['id']): + raise p.toolkit.ValidationError({ + 'fields': '{0} is not a valid field name'.format(field['id']) + }) def create_table(context, data_dict): - '''create table from combination of fields and first row of data''' + 'Create table from combination of fields and first row of data.' check_fields(context, data_dict.get('fields')) - pass + + create_string = 'create table "{0}" ('.format(data_dict['resource_id']) + + # add datastore fields: _id and _full_text + create_string += '_id serial primary key, ' + create_string += '_full_text tsvector, ' + + # add fields + for field in data_dict.get('fields'): + create_string += '"{0}" {1}, '.format(field['id'], field['type']) + + # check first row of data for additional fields + field_ids = [field['id'] for field in data_dict.get('fields', [])] + records = data_dict.get('records') + if records: + extra_field_ids = records[0].keys() + for field_id in extra_field_ids: + if not field_id in field_ids: + field_type = _guess_type(records[0][field_id]) + create_string += '"{0}" {1}, '.format(field_id, field_type) + + # remove last 2 characters (a comma and a space) + # and close the create table statement + create_string = create_string[0:len(create_string) - 2] + create_string += ');' + + context['connection'].execute(create_string) def alter_table(context, data_dict): '''alter table from combination of fields and first row of data''' check_fields(context, data_dict.get('fields')) - pass def insert_data(context, data_dict): @@ -99,6 +158,7 @@ def create(context, data_dict): else: alter_table(context, data_dict) insert_data(context, data_dict) + return data_dict except: trans.rollback() raise diff --git a/ckanext/datastore/logic/action/create.py b/ckanext/datastore/logic/action/create.py index f2a59d144b5..879ec7d32e6 100644 --- a/ckanext/datastore/logic/action/create.py +++ b/ckanext/datastore/logic/action/create.py @@ -44,7 +44,4 @@ def datastore_create(context, data_dict): data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] - try: - return db.create(context, data_dict) - except db.InvalidType as e: - raise p.toolkit.ValidationError({'fields': e.message}) + return db.create(context, data_dict) diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index efa98e3788a..271bc5222b5 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -20,30 +20,45 @@ def setup_class(cls): def teardown_class(cls): model.repo.rebuild_db() + def test_create_requires_auth(self): + resource = model.Package.get('annakarenina').resources[0] + data = { + 'resource_id': resource.id + } + postparams = '%s=1' % json.dumps(data) + res = self.app.post('/api/action/datastore_create', params=postparams, + status=403) + res_dict = json.loads(res.body) + assert res_dict['success'] is False + def test_create_empty_fails(self): postparams = '%s=1' % json.dumps({}) + auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_create', params=postparams, - status=409) + extra_environ=auth, status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False - def test_create_requires_auth(self): + def test_create_invalid_field_type(self): resource = model.Package.get('annakarenina').resources[0] data = { - 'resource_id': resource.id + 'resource_id': resource.id, + 'fields': [{'id': 'book', 'type': 'INVALID'}, + {'id': 'author', 'type': 'INVALID'}] } postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_create', params=postparams, - status=403) + extra_environ=auth, status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False - def test_create_invalid_field(self): + def test_create_invalid_field_name(self): resource = model.Package.get('annakarenina').resources[0] data = { 'resource_id': resource.id, - 'fields': [{'id': 'book', 'label': 'Name', 'type': 'INVALID'}, - {'id': 'author', 'label': 'Author ', 'type': 'INVALID'}] + 'fields': [{'id': '_book', 'type': 'text'}, + {'id': '_author', 'type': 'text'}] } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} @@ -52,12 +67,23 @@ def test_create_invalid_field(self): res_dict = json.loads(res.body) assert res_dict['success'] is False + data = { + 'resource_id': resource.id, + 'fields': [{'id': '"book"', 'type': 'text'}, + {'id': '"author', 'type': 'text'}] + } + postparams = '%s=1' % json.dumps(data) + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth, status=409) + res_dict = json.loads(res.body) + assert res_dict['success'] is False + def test_create_basic(self): resource = model.Package.get('annakarenina').resources[0] data = { 'resource_id': resource.id, - 'fields': [{'id': 'book', 'label': 'Name', 'type': 'text'}, - {'id': 'author', 'label': 'Author ', 'type': 'text'}], + 'fields': [{'id': 'book', 'type': 'text'}, + {'id': 'author', 'type': 'text'}], 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, {'book': 'warandpeace', 'author': 'tolstoy'}] } From 3a91c400bbedfb601cd7a616eadd16d8125d6c5f Mon Sep 17 00:00:00 2001 From: John Glover Date: Wed, 25 Jul 2012 15:41:33 +0100 Subject: [PATCH 007/130] [#2733] Bug fix: commit transaction. --- ckanext/datastore/db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index e388bd3694e..c8c119cf69f 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -158,6 +158,7 @@ def create(context, data_dict): else: alter_table(context, data_dict) insert_data(context, data_dict) + trans.commit() return data_dict except: trans.rollback() From 5ca857cabe6ef95e128173649af0b2cd450e7899 Mon Sep 17 00:00:00 2001 From: John Glover Date: Wed, 25 Jul 2012 16:42:18 +0100 Subject: [PATCH 008/130] [#2733] Add insert_data function and update tests. --- ckanext/datastore/db.py | 56 ++++++++++++++++++++++- ckanext/datastore/tests/test_datastore.py | 25 ++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index c8c119cf69f..eb70e5b5505 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -64,6 +64,20 @@ def _guess_type(field): return 'text' +def _get_fields(context, data_dict): + fields = [] + all_fields = context['connection'].execute( + 'select * from "{0}" limit 1'.format(data_dict['resource_id']) + ) + for field in all_fields.cursor.description: + if not field[0].startswith('_'): + fields.append({ + 'name': field[0], + 'type': _get_type(context, field[1]) + }) + return fields + + def check_fields(context, fields): 'Check if field types are valid.' _cache_types(context) @@ -117,7 +131,47 @@ def alter_table(context, data_dict): def insert_data(context, data_dict): '''insert all data from records''' - pass + if not data_dict.get('records'): + return + + fields = _get_fields(context, data_dict) + insert_string = '' + + for record in data_dict['records']: + # check that number of record values is correct + # TODO: is this necessary? + if not len(record.keys()) == len(fields): + error_msg = 'Field count ({0}) does not match table ({1})'.format( + len(record.keys()), len(fields) + ) + raise p.toolkit.ValidationError({ + 'records': error_msg + }) + + insert_string += 'insert into "{0}" ('.format(data_dict['resource_id']) + + for field in fields: + insert_string += '"{0}", '.format(field['name']) + insert_string = insert_string[0:len(insert_string) - 2] + insert_string += ') ' + + insert_string += 'values (' + + for field in fields: + if not field['name'] in record: + raise p.toolkit.ValidationError({ + 'records': 'Field {0} not found'.format(field['name']) + }) + + if field['type'] == 'text': + insert_string += "'{0}', ".format(record[field['name']]) + else: + insert_string += '{0}, '.format(record[field['name']]) + + insert_string = insert_string[0:len(insert_string) - 2] + insert_string += ');' + + context['connection'].execute(insert_string) def create(context, data_dict): diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index 271bc5222b5..adfaa873d3b 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -78,6 +78,23 @@ def test_create_invalid_field_name(self): res_dict = json.loads(res.body) assert res_dict['success'] is False + def test_create_invalid_record_field(self): + resource = model.Package.get('annakarenina').resources[0] + data = { + 'resource_id': resource.id, + 'fields': [{'id': 'book', 'type': 'text'}, + {'id': 'author', 'type': 'text'}], + 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, + {'book': 'warandpeace', 'published': '1869'}] + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth, status=409) + res_dict = json.loads(res.body) + + assert res_dict['success'] is False + def test_create_basic(self): resource = model.Package.get('annakarenina').resources[0] data = { @@ -97,3 +114,11 @@ def test_create_basic(self): assert res_dict['result']['resource_id'] == data['resource_id'] assert res_dict['result']['fields'] == data['fields'] assert res_dict['result']['records'] == data['records'] + + c = model.Session.connection() + results = c.execute('select * from "{0}"'.format(resource.id)) + + assert results.rowcount == 2 + for i, row in enumerate(results): + assert data['records'][i]['book'] == row['book'] + assert data['records'][i]['author'] == row['author'] From 0f3319b2fdf43ff25e56a45f9c06bafba4abc6ca Mon Sep 17 00:00:00 2001 From: John Glover Date: Wed, 25 Jul 2012 16:54:03 +0100 Subject: [PATCH 009/130] [#2733] Refactor: tidy up insert sql statement. --- ckanext/datastore/db.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index eb70e5b5505..4cbee51c0b2 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -135,7 +135,6 @@ def insert_data(context, data_dict): return fields = _get_fields(context, data_dict) - insert_string = '' for record in data_dict['records']: # check that number of record values is correct @@ -148,14 +147,8 @@ def insert_data(context, data_dict): 'records': error_msg }) - insert_string += 'insert into "{0}" ('.format(data_dict['resource_id']) - - for field in fields: - insert_string += '"{0}", '.format(field['name']) - insert_string = insert_string[0:len(insert_string) - 2] - insert_string += ') ' - - insert_string += 'values (' + sql_columns = ", ".join(['"%s"' % f['name'] for f in fields]) + sql_values = [] for field in fields: if not field['name'] in record: @@ -164,14 +157,19 @@ def insert_data(context, data_dict): }) if field['type'] == 'text': - insert_string += "'{0}', ".format(record[field['name']]) + sql_values.append("'{0}'".format(record[field['name']])) else: - insert_string += '{0}, '.format(record[field['name']]) + sql_values.append('{0}'.format(record[field['name']])) - insert_string = insert_string[0:len(insert_string) - 2] - insert_string += ');' + sql_values = ", ".join(['%s' % v for v in sql_values]) + + sql_string = 'insert into "{0}" ({1}) values ({2});'.format( + data_dict['resource_id'], + sql_columns, + sql_values + ) - context['connection'].execute(insert_string) + context['connection'].execute(sql_string) def create(context, data_dict): From a5d44a1dcfd45f031bb9365258def080fd02dec5 Mon Sep 17 00:00:00 2001 From: John Glover Date: Wed, 25 Jul 2012 17:12:17 +0100 Subject: [PATCH 010/130] [#2733] Refactor: tidy up create sql statement. --- ckanext/datastore/db.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 4cbee51c0b2..18d0216fe3e 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -96,32 +96,35 @@ def create_table(context, data_dict): 'Create table from combination of fields and first row of data.' check_fields(context, data_dict.get('fields')) - create_string = 'create table "{0}" ('.format(data_dict['resource_id']) - - # add datastore fields: _id and _full_text - create_string += '_id serial primary key, ' - create_string += '_full_text tsvector, ' - - # add fields - for field in data_dict.get('fields'): - create_string += '"{0}" {1}, '.format(field['id'], field['type']) + datastore_fields = [ + {'id': '_id', 'type': 'serial primary key'}, + {'id': '_full_text', 'type': 'tsvector'}, + ] # check first row of data for additional fields + extra_fields = [] field_ids = [field['id'] for field in data_dict.get('fields', [])] records = data_dict.get('records') + if records: extra_field_ids = records[0].keys() for field_id in extra_field_ids: if not field_id in field_ids: - field_type = _guess_type(records[0][field_id]) - create_string += '"{0}" {1}, '.format(field_id, field_type) + extra_fields.append({ + 'id': field_id, + 'type': _guess_type(records[0][field_id]) + }) + + fields = datastore_fields + data_dict.get('fields', []) + extra_fields + sql_fields = ", ".join(['"{0}" {1}'.format(f['id'], f['type']) + for f in fields]) - # remove last 2 characters (a comma and a space) - # and close the create table statement - create_string = create_string[0:len(create_string) - 2] - create_string += ');' + sql_string = 'create table "{0}" ({1});'.format( + data_dict['resource_id'], + sql_fields + ) - context['connection'].execute(create_string) + context['connection'].execute(sql_string) def alter_table(context, data_dict): From 981cf79e83153bdc4cd75d288939c70077d2b595 Mon Sep 17 00:00:00 2001 From: kindly Date: Sun, 29 Jul 2012 01:22:53 +0100 Subject: [PATCH 011/130] create finished apart from table update --- ckanext/datastore/db.py | 128 +++++++++++++++------- ckanext/datastore/logic/action/create.py | 8 -- ckanext/datastore/logic/auth/create.py | 3 + ckanext/datastore/tests/test_datastore.py | 84 +++++++++++++- 4 files changed, 173 insertions(+), 50 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 18d0216fe3e..273557f9e8b 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1,10 +1,16 @@ import sqlalchemy import ckan.plugins as p +import json +import datetime _pg_types = {} _type_names = set() _engines = {} +_iso_formats = ['%Y-%m-%d', + '%Y-%m-%d %H:%M:%S', + '%Y-%m-%dT%H:%M:%S'] + def _is_valid_field_name(name): ''' @@ -23,7 +29,7 @@ def _get_engine(context, data_dict): engine = _engines.get(connection_url) if not engine: - engine = sqlalchemy.create_engine(connection_url) + engine = sqlalchemy.create_engine(connection_url, echo=True) _engines[connection_url] = engine return engine @@ -55,13 +61,19 @@ def _guess_type(field): data_types.discard(data_type) if not data_types: break - if int in data_types: return 'integer' elif float in data_types: return 'numeric' - else: - return 'text' + + ##try iso dates + for format in _iso_formats: + try: + datetime.datetime.strptime(field, format) + return 'timestamp' + except ValueError: + continue + return 'text' def _get_fields(context, data_dict): @@ -72,17 +84,29 @@ def _get_fields(context, data_dict): for field in all_fields.cursor.description: if not field[0].startswith('_'): fields.append({ - 'name': field[0], + 'id': field[0], 'type': _get_type(context, field[1]) }) return fields +def json_get_values(obj, current_list=None): + if current_list is None: + current_list = [] + if isinstance(obj, basestring): + current_list.append(obj) + if isinstance(obj, list): + for item in obj: + json_get_values(item, current_list) + if isinstance(obj, dict): + for item in dict.values(): + json_get_values(item, current_list) + return current_list def check_fields(context, fields): 'Check if field types are valid.' _cache_types(context) for field in fields: - if not field['type'] in _type_names: + if field.get('type') and not field['type'] in _type_names: raise p.toolkit.ValidationError({ 'fields': '{0} is not a valid field type'.format(field['type']) }) @@ -103,19 +127,34 @@ def create_table(context, data_dict): # check first row of data for additional fields extra_fields = [] + supplied_fields = data_dict.get('fields', []) field_ids = [field['id'] for field in data_dict.get('fields', [])] records = data_dict.get('records') + # if type is field is not given try and guess or throw an error + for field in supplied_fields: + if 'type' not in field: + if not records or field['id'] not in records[0]: + raise p.toolkit.ValidationError({ + 'fields': '{} type not guessable'.format(field['id']) + }) + field['type'] = _guess_type(records[0][field['id']]) + if records: - extra_field_ids = records[0].keys() - for field_id in extra_field_ids: + # check record for sanity + if not isinstance(records[0], dict): + raise p.toolkit.ValidationError({ + 'records': 'The first row is not a json object' + }) + supplied_field_ids = records[0].keys() + for field_id in supplied_field_ids: if not field_id in field_ids: extra_fields.append({ 'id': field_id, 'type': _guess_type(records[0][field_id]) }) - fields = datastore_fields + data_dict.get('fields', []) + extra_fields + fields = datastore_fields + supplied_fields + extra_fields sql_fields = ", ".join(['"{0}" {1}'.format(f['id'], f['type']) for f in fields]) @@ -130,6 +169,7 @@ def create_table(context, data_dict): def alter_table(context, data_dict): '''alter table from combination of fields and first row of data''' check_fields(context, data_dict.get('fields')) + fields = _get_fields(context, data_dict) def insert_data(context, data_dict): @@ -138,41 +178,51 @@ def insert_data(context, data_dict): return fields = _get_fields(context, data_dict) + field_names = [field['id'] for field in fields] + ['_full_text'] + sql_columns = ", ".join(['"%s"' % name for name in field_names]) + + rows = [] + + ## clean up and validate data + for num, record in enumerate(data_dict['records']): - for record in data_dict['records']: - # check that number of record values is correct - # TODO: is this necessary? - if not len(record.keys()) == len(fields): - error_msg = 'Field count ({0}) does not match table ({1})'.format( - len(record.keys()), len(fields) - ) + # check record for sanity + if not isinstance(record, dict): raise p.toolkit.ValidationError({ - 'records': error_msg + 'records': 'row {} is not a json object'.format(num) + }) + ## check for extra fields in data + extra_keys = set(record.keys()) - set(field_names) + if extra_keys: + raise p.toolkit.ValidationError({ + 'records': 'row {} has extra keys "{}"'.format( + num, + ', '.join(list(extra_keys)) + ) }) - sql_columns = ", ".join(['"%s"' % f['name'] for f in fields]) - sql_values = [] - + full_text = [] + row = [] for field in fields: - if not field['name'] in record: - raise p.toolkit.ValidationError({ - 'records': 'Field {0} not found'.format(field['name']) - }) - - if field['type'] == 'text': - sql_values.append("'{0}'".format(record[field['name']])) - else: - sql_values.append('{0}'.format(record[field['name']])) - - sql_values = ", ".join(['%s' % v for v in sql_values]) - - sql_string = 'insert into "{0}" ({1}) values ({2});'.format( - data_dict['resource_id'], - sql_columns, - sql_values - ) + value = record.get(field['id']) + if isinstance(value, (dict, list)): + full_text.extend(json_get_values(value)) + value = json.dumps(value) + elif field['type'].lower() == 'text' and value: + full_text.append(value) + row.append(value) + + row.append(' '.join(full_text)) + rows.append(row) + + sql_string = 'insert into "{0}" ({1}) values ({2});'.format( + data_dict['resource_id'], + sql_columns, + ', '.join(['%s' for field in field_names]) + ) - context['connection'].execute(sql_string) + + context['connection'].execute(sql_string, rows) def create(context, data_dict): @@ -187,7 +237,7 @@ def create(context, data_dict): add extra information for certain columns or to explicitly define ordering. - eg [{"id": "dob", "label": ""Date of Birth", + eg [datetime.datetime.strptime(field, format){"id": "dob", "label": ""Date of Birth", "type": "timestamp" ,"concept": "day"}, {"name": "some_stuff": ..]. diff --git a/ckanext/datastore/logic/action/create.py b/ckanext/datastore/logic/action/create.py index 879ec7d32e6..ec29c1c0cf4 100644 --- a/ckanext/datastore/logic/action/create.py +++ b/ckanext/datastore/logic/action/create.py @@ -32,16 +32,8 @@ def datastore_create(context, data_dict): _check_access('datastore_create', context, data_dict) - # TODO: remove this check for resource ID when the resource_id_exists - # validator has been created. _get_or_bust(data_dict, 'resource_id') - schema = ckanext.datastore.logic.schema.default_datastore_create_schema() - data, errors = _validate(data_dict, schema, context) - if errors: - model.Session.rollback() - raise p.toolkit.ValidationError(errors) - data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] return db.create(context, data_dict) diff --git a/ckanext/datastore/logic/auth/create.py b/ckanext/datastore/logic/auth/create.py index d5db8d0ebdf..2c51e55055b 100644 --- a/ckanext/datastore/logic/auth/create.py +++ b/ckanext/datastore/logic/auth/create.py @@ -1,4 +1,7 @@ import ckan.plugins as p +import ckan.logic as logic + +_check_access = logic.check_access def datastore_create(context, data_dict): diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index adfaa873d3b..b68c9c95fa4 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -3,6 +3,7 @@ import ckan.lib.create_test_data as ctd import ckan.model as model import ckan.tests as tests +import ckanext.datastore.db as db class TestDatastore(tests.WsgiAppCase): @@ -92,9 +93,42 @@ def test_create_invalid_record_field(self): res = self.app.post('/api/action/datastore_create', params=postparams, extra_environ=auth, status=409) res_dict = json.loads(res.body) + assert res_dict['success'] is False + + def test_bad_records(self): + resource = model.Package.get('annakarenina').resources[0] + data = { + 'resource_id': resource.id, + 'fields': [{'id': 'book', 'type': 'text'}, + {'id': 'author', 'type': 'text'}], + 'records': ['bad'] # treat author as null + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth, status=409) + res_dict = json.loads(res.body) + + assert res_dict['success'] is False + + resource = model.Package.get('annakarenina').resources[0] + data = { + 'resource_id': resource.id, + 'fields': [{'id': 'book', 'type': 'text'}, + {'id': 'author', 'type': 'text'}], + 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, + [], + {'book': 'warandpeace'}] # treat author as null + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth, status=409) + res_dict = json.loads(res.body) assert res_dict['success'] is False + def test_create_basic(self): resource = model.Package.get('annakarenina').resources[0] data = { @@ -102,7 +136,8 @@ def test_create_basic(self): 'fields': [{'id': 'book', 'type': 'text'}, {'id': 'author', 'type': 'text'}], 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, - {'book': 'warandpeace', 'author': 'tolstoy'}] + {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']}, + {'book': 'warandpeace'}] # treat author as null } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} @@ -118,7 +153,50 @@ def test_create_basic(self): c = model.Session.connection() results = c.execute('select * from "{0}"'.format(resource.id)) + assert results.rowcount == 3 + for i, row in enumerate(results): + assert data['records'][i].get('book') == row['book'] + assert (data['records'][i].get('author') == row['author'] + or data['records'][i].get('author') == json.loads(row['author'])) + + results = c.execute('''select * from "{0}" where _full_text @@ 'warandpeace' '''.format(resource.id)) + assert results.rowcount == 1 + + results = c.execute('''select * from "{0}" where _full_text @@ 'tolstoy' '''.format(resource.id)) assert results.rowcount == 2 + + def test_guess_types(self): + resource = model.Package.get('annakarenina').resources[1] + data = { + 'resource_id': resource.id, + 'fields': [{'id': 'author', 'type': 'text'}, + {'id': 'count'}, + {'id': 'book'}, + {'id': 'date'}], + 'records': [{'book': 'annakarenina', 'author': 'tolstoy', 'count': 1, + 'date': '2005-12-01', 'count2' : 2}, + {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']}, + {'book': 'warandpeace'}] # treat author as null + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + + c = model.Session.connection() + results = c.execute('''select * from "{0}" '''.format(resource.id)) + + types = [db._pg_types[field[1]] for field in results.cursor.description] + + assert types == [u'int4', u'tsvector', u'text', u'int4', u'text', u'timestamp', u'int4'], types + + assert results.rowcount == 3 for i, row in enumerate(results): - assert data['records'][i]['book'] == row['book'] - assert data['records'][i]['author'] == row['author'] + assert data['records'][i].get('book') == row['book'] + assert (data['records'][i].get('author') == row['author'] + or data['records'][i].get('author') == json.loads(row['author'])) + + + + From ac3d2610ac4d26603f370d695f501e7bd02ed0ef Mon Sep 17 00:00:00 2001 From: kindly Date: Sun, 29 Jul 2012 01:47:22 +0100 Subject: [PATCH 012/130] change auth to resource update --- ckanext/datastore/logic/action/create.py | 6 ++++++ ckanext/datastore/logic/auth/create.py | 14 +++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ckanext/datastore/logic/action/create.py b/ckanext/datastore/logic/action/create.py index ec29c1c0cf4..de518ac0ad7 100644 --- a/ckanext/datastore/logic/action/create.py +++ b/ckanext/datastore/logic/action/create.py @@ -34,6 +34,12 @@ def datastore_create(context, data_dict): _get_or_bust(data_dict, 'resource_id') + #schema = ckanext.datastore.logic.schema.default_datastore_create_schema() + #data, errors = _validate(data_dict, schema, context) + #if errors: + # model.Session.rollback() + # raise p.toolkit.ValidationError(errors) + data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] return db.create(context, data_dict) diff --git a/ckanext/datastore/logic/auth/create.py b/ckanext/datastore/logic/auth/create.py index 2c51e55055b..a13210a82da 100644 --- a/ckanext/datastore/logic/auth/create.py +++ b/ckanext/datastore/logic/auth/create.py @@ -1,15 +1,15 @@ import ckan.plugins as p import ckan.logic as logic +from ckan.lib.base import _ _check_access = logic.check_access - def datastore_create(context, data_dict): - model = context['model'] - user = context['user'] - userobj = model.User.get(user) + data_dict['id'] = data_dict.get('resource_id') + + authorized = _check_access('resource_update', context, data_dict) - if userobj: + if not authorized: + return {'success': False, 'msg': _('User %s not authorized to read edit %s') % (str(user), resource.id)} + else: return {'success': True} - return {'success': False, - 'msg': p.toolkit._('You must be logged in to use the datastore.')} From 1195ca73ab20727e29bf239f716ca50bb41bca9b Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 30 Jul 2012 01:12:38 +0100 Subject: [PATCH 013/130] [2722] finish datastore create --- ckanext/datastore/db.py | 56 ++++++++- ckanext/datastore/tests/test_datastore.py | 132 +++++++++++++++++++++- 2 files changed, 180 insertions(+), 8 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 273557f9e8b..5e16314783c 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -118,7 +118,6 @@ def check_fields(context, fields): def create_table(context, data_dict): 'Create table from combination of fields and first row of data.' - check_fields(context, data_dict.get('fields')) datastore_fields = [ {'id': '_id', 'type': 'serial primary key'}, @@ -128,6 +127,7 @@ def create_table(context, data_dict): # check first row of data for additional fields extra_fields = [] supplied_fields = data_dict.get('fields', []) + check_fields(context, supplied_fields) field_ids = [field['id'] for field in data_dict.get('fields', [])] records = data_dict.get('records') @@ -168,8 +168,56 @@ def create_table(context, data_dict): def alter_table(context, data_dict): '''alter table from combination of fields and first row of data''' - check_fields(context, data_dict.get('fields')) - fields = _get_fields(context, data_dict) + supplied_fields = data_dict.get('fields', []) + current_fields = _get_fields(context, data_dict) + if not supplied_fields: + supplied_fields = current_fields + check_fields(context, supplied_fields) + field_ids = [field['id'] for field in supplied_fields] + records = data_dict.get('records') + new_fields = [] + + for num, field in enumerate(supplied_fields): + # check to see if field definition is the same or an + # extension of current fields + if num < len(current_fields): + if field['id'] <> current_fields[num]['id']: + raise p.toolkit.ValidationError({ + 'fields': ('Supplied field "{}" not ' + 'present or in wrong order').format(field['id']) + }) + ## no need to check type as field already defined. + continue + + if 'type' not in field: + if not records or field['id'] not in records[0]: + raise p.toolkit.ValidationError({ + 'fields': '{} type not guessable'.format(field['id']) + }) + field['type'] = _guess_type(records[0][field['id']]) + new_fields.append(field) + + if records: + # check record for sanity + if not isinstance(records[0], dict): + raise p.toolkit.ValidationError({ + 'records': 'The first row is not a json object' + }) + supplied_field_ids = records[0].keys() + for field_id in supplied_field_ids: + if not field_id in field_ids: + new_fields.append({ + 'id': field_id, + 'type': _guess_type(records[0][field_id]) + }) + + + for field in new_fields: + sql = 'alter table "{}" add "{}" {}'.format( + data_dict['resource_id'], + field['id'], + field['type']) + context['connection'].execute(sql) def insert_data(context, data_dict): @@ -221,7 +269,7 @@ def insert_data(context, data_dict): ', '.join(['%s' for field in field_names]) ) - + context['connection'].execute(sql_string, rows) diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index b68c9c95fa4..95c87552ef4 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -156,7 +156,7 @@ def test_create_basic(self): assert results.rowcount == 3 for i, row in enumerate(results): assert data['records'][i].get('book') == row['book'] - assert (data['records'][i].get('author') == row['author'] + assert (data['records'][i].get('author') == row['author'] or data['records'][i].get('author') == json.loads(row['author'])) results = c.execute('''select * from "{0}" where _full_text @@ 'warandpeace' '''.format(resource.id)) @@ -164,6 +164,68 @@ def test_create_basic(self): results = c.execute('''select * from "{0}" where _full_text @@ 'tolstoy' '''.format(resource.id)) assert results.rowcount == 2 + model.Session.remove() + + ####### insert again simple + data2 = { + 'resource_id': resource.id, + 'records': [{'book': 'hagji murat', 'author': 'tolstoy'}] + } + + postparams = '%s=1' % json.dumps(data2) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + + assert res_dict['success'] is True + + c = model.Session.connection() + results = c.execute('select * from "{0}"'.format(resource.id)) + + assert results.rowcount == 4 + + all_data = data['records'] + data2['records'] + for i, row in enumerate(results): + assert all_data[i].get('book') == row['book'] + assert (all_data[i].get('author') == row['author'] + or all_data[i].get('author') == json.loads(row['author'])) + + results = c.execute('''select * from "{0}" where _full_text @@ 'tolstoy' '''.format(resource.id)) + assert results.rowcount == 3 + model.Session.remove() + + ####### insert again extra field + data3 = { + 'resource_id': resource.id, + 'records': [{'book': 'crime and punsihment', + 'author': 'dostoevsky', 'rating': 'good'}] + } + + postparams = '%s=1' % json.dumps(data3) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + + assert res_dict['success'] is True + + c = model.Session.connection() + results = c.execute('select * from "{0}"'.format(resource.id)) + + assert results.rowcount == 5 + + all_data = data['records'] + data2['records'] + data3['records'] + print all_data + for i, row in enumerate(results): + assert all_data[i].get('book') == row['book'], (i, all_data[i].get('book'), row['book']) + assert (all_data[i].get('author') == row['author'] + or all_data[i].get('author') == json.loads(row['author'])) + + results = c.execute('''select * from "{0}" where _full_text @@ 'dostoevsky' '''.format(resource.id)) + assert results.rowcount == 2 + model.Session.remove() + def test_guess_types(self): resource = model.Package.get('annakarenina').resources[1] @@ -186,7 +248,7 @@ def test_guess_types(self): c = model.Session.connection() results = c.execute('''select * from "{0}" '''.format(resource.id)) - + types = [db._pg_types[field[1]] for field in results.cursor.description] assert types == [u'int4', u'tsvector', u'text', u'int4', u'text', u'timestamp', u'int4'], types @@ -194,9 +256,71 @@ def test_guess_types(self): assert results.rowcount == 3 for i, row in enumerate(results): assert data['records'][i].get('book') == row['book'] - assert (data['records'][i].get('author') == row['author'] + assert (data['records'][i].get('author') == row['author'] or data['records'][i].get('author') == json.loads(row['author'])) - + model.Session.remove() + ### extend types + data = { + 'resource_id': resource.id, + 'fields': [{'id': 'author', 'type': 'text'}, + {'id': 'count'}, + {'id': 'book'}, + {'id': 'date'}, + {'id': 'count2'}, + {'id': 'extra', 'type':'text'}, + {'id': 'date2'}, + ], + 'records': [{'book': 'annakarenina', 'author': 'tolstoy', 'count': 1, + 'date': '2005-12-01', 'count2' : 2, 'count3': 432, + 'date2': '2005-12-01'}] + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + + c = model.Session.connection() + results = c.execute('''select * from "{0}" '''.format(resource.id)) + + types = [db._pg_types[field[1]] for field in results.cursor.description] + + assert types == [u'int4', #id + u'tsvector', #fulltext + u'text', #author + u'int4', #count + u'text', #book + u'timestamp', #date + u'int4', #count2 + u'text', #extra + u'timestamp', #date2 + u'int4', #count3 + ], types + + ### fields resupplied in wrong order + + data = { + 'resource_id': resource.id, + 'fields': [{'id': 'author', 'type': 'text'}, + {'id': 'count'}, + {'id': 'date'}, ## date and book in wrong order + {'id': 'book'}, + {'id': 'count2'}, + {'id': 'extra', 'type':'text'}, + {'id': 'date2'}, + ], + 'records': [{'book': 'annakarenina', 'author': 'tolstoy', 'count': 1, + 'date': '2005-12-01', 'count2' : 2, 'count3': 432, + 'date2': '2005-12-01'}] + } + + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth, status=409) + res_dict = json.loads(res.body) + + assert res_dict['success'] is False From 785cd095c655d8922992d47bf5df78bc2bf93552 Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 30 Jul 2012 02:21:26 +0100 Subject: [PATCH 014/130] [2733] start delete --- ckanext/datastore/db.py | 60 ++++++++++++++++++++++++++ ckanext/datastore/logic/auth/create.py | 4 +- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 5e16314783c..5d4cbfa3772 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -272,6 +272,37 @@ def insert_data(context, data_dict): context['connection'].execute(sql_string, rows) +def delete_data(context, data_dict): + filter = data_dict.get('filter') + + if not isinstance(filter, dict): + raise p.toolkit.ValidationError({ + 'filter': 'Not a json object'} + ) + + fields = _get_fields(context, data_dict) + field_ids = set([field['id'] for field in fields]) + + where_clauses = [] + values = [] + for field, value in filter.iteritems(): + if field not in field_ids: + raise p.toolkit.ValidationError({ + 'filter': 'field "{}" not in table'} + ) + where_clauses.append('"{}" = %s'.format(field)) + values.append(value) + + where_clause = ' and '.join(where_clauses) + + context['connection'].execute( + 'delete from "{}" where {}'.format( + data_dict['resource_id'], + where_clause + ), + values + ) + def create(context, data_dict): ''' @@ -318,3 +349,32 @@ def create(context, data_dict): raise finally: context['connection'].close() + +def delete(context, data_dict): + engine = _get_engine(context, data_dict) + context['connection'] = engine.connect() + + try: + # check if table existes + trans = context['connection'].begin() + result = context['connection'].execute( + 'select * from pg_tables where tablename = %s', + data_dict['resource_id'] + ).fetchone() + if not result: + raise p.toolkit.ValidationError({ + 'resource_id': 'table for {0} does not exist'.format( + data_dict['resource_id']) + }) + if not 'filter' in data_dict: + context['connection'].execute('drop table "{}"'.format(data_dict)) + else: + delete_data(context, data_dict) + + trans.commit() + return data_dict + except: + trans.rollback() + raise + finally: + context['connection'].close() diff --git a/ckanext/datastore/logic/auth/create.py b/ckanext/datastore/logic/auth/create.py index a13210a82da..f833df841c0 100644 --- a/ckanext/datastore/logic/auth/create.py +++ b/ckanext/datastore/logic/auth/create.py @@ -6,10 +6,12 @@ def datastore_create(context, data_dict): data_dict['id'] = data_dict.get('resource_id') + user = context.get('user') authorized = _check_access('resource_update', context, data_dict) if not authorized: - return {'success': False, 'msg': _('User %s not authorized to read edit %s') % (str(user), resource.id)} + return {'success': False, + 'msg': _('User %s not authorized to read edit %s') % (str(user), data_dict['id'])} else: return {'success': True} From 5f9cae9ba7dc3a009f66fef27b88f23227d16306 Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 30 Jul 2012 10:00:53 +0100 Subject: [PATCH 015/130] [2733] add delete logic files --- ckanext/datastore/db.py | 2 +- ckanext/datastore/logic/action/delete.py | 37 ++++++++++++++++++++++++ ckanext/datastore/logic/auth/delete.py | 17 +++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 ckanext/datastore/logic/action/delete.py create mode 100644 ckanext/datastore/logic/auth/delete.py diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 5d4cbfa3772..061deb89e0a 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -363,7 +363,7 @@ def delete(context, data_dict): ).fetchone() if not result: raise p.toolkit.ValidationError({ - 'resource_id': 'table for {0} does not exist'.format( + 'resource_id': 'table for resource {0} does not exist'.format( data_dict['resource_id']) }) if not 'filter' in data_dict: diff --git a/ckanext/datastore/logic/action/delete.py b/ckanext/datastore/logic/action/delete.py new file mode 100644 index 00000000000..5243bf3b7f9 --- /dev/null +++ b/ckanext/datastore/logic/action/delete.py @@ -0,0 +1,37 @@ +import logging +import pylons +import ckan.logic as logic +import ckan.logic.action +import ckan.lib.dictization +import ckan.plugins as p +import ckanext.datastore.logic.schema +import ckanext.datastore.db as db + +log = logging.getLogger(__name__) + +_validate = ckan.lib.navl.dictization_functions.validate +_check_access = logic.check_access +_get_or_bust = logic.get_or_bust + + +def datastore_delete(context, data_dict): + '''Adds a new table to the datastore. + + :param resource_id: resource id that the data is going to be stored under. + :type resource_id: string + :param filter: Filter to do deleting on over eg {'name': 'fred'} if missing + delete whole table + + :returns: original filters sent. + :rtype: dictionary + + ''' + model = _get_or_bust(context, 'model') + + _check_access('datastore_delete', context, data_dict) + + _get_or_bust(data_dict, 'resource_id') + + data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] + + return db.delete(context, data_dict) diff --git a/ckanext/datastore/logic/auth/delete.py b/ckanext/datastore/logic/auth/delete.py new file mode 100644 index 00000000000..2e3992cf873 --- /dev/null +++ b/ckanext/datastore/logic/auth/delete.py @@ -0,0 +1,17 @@ +import ckan.plugins as p +import ckan.logic as logic +from ckan.lib.base import _ + +_check_access = logic.check_access + +def datastore_delete(context, data_dict): + data_dict['id'] = data_dict.get('resource_id') + user = context.get('user') + + authorized = _check_access('resource_update', context, data_dict) + + if not authorized: + return {'success': False, + 'msg': _('User %s not authorized to read edit %s') % (str(user), data_dict['id'])} + else: + return {'success': True} From 8a93747b541aac6a3d87f04c96c89af67919a6cd Mon Sep 17 00:00:00 2001 From: John Glover Date: Mon, 30 Jul 2012 11:13:10 +0100 Subject: [PATCH 016/130] [#2733] Datastore PEP8 cleanup. --- ckanext/datastore/db.py | 13 +++--- ckanext/datastore/logic/action/create.py | 14 ++----- ckanext/datastore/logic/auth/create.py | 5 ++- ckanext/datastore/tests/test_datastore.py | 49 +++++++++++------------ 4 files changed, 37 insertions(+), 44 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 061deb89e0a..be077274987 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -89,6 +89,7 @@ def _get_fields(context, data_dict): }) return fields + def json_get_values(obj, current_list=None): if current_list is None: current_list = [] @@ -102,6 +103,7 @@ def json_get_values(obj, current_list=None): json_get_values(item, current_list) return current_list + def check_fields(context, fields): 'Check if field types are valid.' _cache_types(context) @@ -181,7 +183,7 @@ def alter_table(context, data_dict): # check to see if field definition is the same or an # extension of current fields if num < len(current_fields): - if field['id'] <> current_fields[num]['id']: + if field['id'] != current_fields[num]['id']: raise p.toolkit.ValidationError({ 'fields': ('Supplied field "{}" not ' 'present or in wrong order').format(field['id']) @@ -211,7 +213,6 @@ def alter_table(context, data_dict): 'type': _guess_type(records[0][field_id]) }) - for field in new_fields: sql = 'alter table "{}" add "{}" {}'.format( data_dict['resource_id'], @@ -269,9 +270,9 @@ def insert_data(context, data_dict): ', '.join(['%s' for field in field_names]) ) - context['connection'].execute(sql_string, rows) + def delete_data(context, data_dict): filter = data_dict.get('filter') @@ -316,9 +317,8 @@ def create(context, data_dict): add extra information for certain columns or to explicitly define ordering. - eg [datetime.datetime.strptime(field, format){"id": "dob", "label": ""Date of Birth", - "type": "timestamp" ,"concept": "day"}, - {"name": "some_stuff": ..]. + eg: [{"id": "dob", "type": "timestamp"}, + {"id": "name", "type": "text"}] A header items values can not be changed after it has been defined nor can the ordering of them be changed. They can be extended though. @@ -350,6 +350,7 @@ def create(context, data_dict): finally: context['connection'].close() + def delete(context, data_dict): engine = _get_engine(context, data_dict) context['connection'] = engine.connect() diff --git a/ckanext/datastore/logic/action/create.py b/ckanext/datastore/logic/action/create.py index de518ac0ad7..58ef8ce96dc 100644 --- a/ckanext/datastore/logic/action/create.py +++ b/ckanext/datastore/logic/action/create.py @@ -3,8 +3,6 @@ import ckan.logic as logic import ckan.logic.action import ckan.lib.dictization -import ckan.plugins as p -import ckanext.datastore.logic.schema import ckanext.datastore.db as db log = logging.getLogger(__name__) @@ -28,17 +26,11 @@ def datastore_create(context, data_dict): :rtype: dictionary ''' - model = _get_or_bust(context, 'model') - - _check_access('datastore_create', context, data_dict) - + _get_or_bust(context, 'model') _get_or_bust(data_dict, 'resource_id') + # TODO: check that resource_id exists in database - #schema = ckanext.datastore.logic.schema.default_datastore_create_schema() - #data, errors = _validate(data_dict, schema, context) - #if errors: - # model.Session.rollback() - # raise p.toolkit.ValidationError(errors) + _check_access('datastore_create', context, data_dict) data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] diff --git a/ckanext/datastore/logic/auth/create.py b/ckanext/datastore/logic/auth/create.py index f833df841c0..f5c881a930c 100644 --- a/ckanext/datastore/logic/auth/create.py +++ b/ckanext/datastore/logic/auth/create.py @@ -1,9 +1,9 @@ -import ckan.plugins as p import ckan.logic as logic from ckan.lib.base import _ _check_access = logic.check_access + def datastore_create(context, data_dict): data_dict['id'] = data_dict.get('resource_id') user = context.get('user') @@ -12,6 +12,7 @@ def datastore_create(context, data_dict): if not authorized: return {'success': False, - 'msg': _('User %s not authorized to read edit %s') % (str(user), data_dict['id'])} + 'msg': _('User %s not authorized to read edit %s') % \ + (str(user), data_dict['id'])} else: return {'success': True} diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index 95c87552ef4..bf5c7c7adf3 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -118,7 +118,7 @@ def test_bad_records(self): {'id': 'author', 'type': 'text'}], 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, [], - {'book': 'warandpeace'}] # treat author as null + {'book': 'warandpeace'}] # treat author as null } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} @@ -128,7 +128,6 @@ def test_bad_records(self): assert res_dict['success'] is False - def test_create_basic(self): resource = model.Package.get('annakarenina').resources[0] data = { @@ -137,7 +136,7 @@ def test_create_basic(self): {'id': 'author', 'type': 'text'}], 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']}, - {'book': 'warandpeace'}] # treat author as null + {'book': 'warandpeace'}] # treat author as null } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} @@ -226,7 +225,6 @@ def test_create_basic(self): assert results.rowcount == 2 model.Session.remove() - def test_guess_types(self): resource = model.Package.get('annakarenina').resources[1] data = { @@ -235,10 +233,10 @@ def test_guess_types(self): {'id': 'count'}, {'id': 'book'}, {'id': 'date'}], - 'records': [{'book': 'annakarenina', 'author': 'tolstoy', 'count': 1, - 'date': '2005-12-01', 'count2' : 2}, + 'records': [{'book': 'annakarenina', 'author': 'tolstoy', + 'count': 1, 'date': '2005-12-01', 'count2': 2}, {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']}, - {'book': 'warandpeace'}] # treat author as null + {'book': 'warandpeace'}] # treat author as null } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} @@ -251,7 +249,8 @@ def test_guess_types(self): types = [db._pg_types[field[1]] for field in results.cursor.description] - assert types == [u'int4', u'tsvector', u'text', u'int4', u'text', u'timestamp', u'int4'], types + assert types == [u'int4', u'tsvector', u'text', u'int4', + u'text', u'timestamp', u'int4'], types assert results.rowcount == 3 for i, row in enumerate(results): @@ -272,9 +271,9 @@ def test_guess_types(self): {'id': 'extra', 'type':'text'}, {'id': 'date2'}, ], - 'records': [{'book': 'annakarenina', 'author': 'tolstoy', 'count': 1, - 'date': '2005-12-01', 'count2' : 2, 'count3': 432, - 'date2': '2005-12-01'}] + 'records': [{'book': 'annakarenina', 'author': 'tolstoy', + 'count': 1, 'date': '2005-12-01', 'count2': 2, + 'count3': 432, 'date2': '2005-12-01'}] } postparams = '%s=1' % json.dumps(data) @@ -288,16 +287,16 @@ def test_guess_types(self): types = [db._pg_types[field[1]] for field in results.cursor.description] - assert types == [u'int4', #id - u'tsvector', #fulltext - u'text', #author - u'int4', #count - u'text', #book - u'timestamp', #date - u'int4', #count2 - u'text', #extra - u'timestamp', #date2 - u'int4', #count3 + assert types == [u'int4', # id + u'tsvector', # fulltext + u'text', # author + u'int4', # count + u'text', # book + u'timestamp', # date + u'int4', # count2 + u'text', # extra + u'timestamp', # date2 + u'int4', # count3 ], types ### fields resupplied in wrong order @@ -306,15 +305,15 @@ def test_guess_types(self): 'resource_id': resource.id, 'fields': [{'id': 'author', 'type': 'text'}, {'id': 'count'}, - {'id': 'date'}, ## date and book in wrong order + {'id': 'date'}, # date and book in wrong order {'id': 'book'}, {'id': 'count2'}, {'id': 'extra', 'type':'text'}, {'id': 'date2'}, ], - 'records': [{'book': 'annakarenina', 'author': 'tolstoy', 'count': 1, - 'date': '2005-12-01', 'count2' : 2, 'count3': 432, - 'date2': '2005-12-01'}] + 'records': [{'book': 'annakarenina', 'author': 'tolstoy', + 'count': 1, 'date': '2005-12-01', 'count2': 2, + 'count3': 432, 'date2': '2005-12-01'}] } postparams = '%s=1' % json.dumps(data) From 5daba25f33b9b7c2021c0a879e65e7be3f81e43c Mon Sep 17 00:00:00 2001 From: John Glover Date: Mon, 30 Jul 2012 11:50:29 +0100 Subject: [PATCH 017/130] [#2733] Fix datastore delete (no filters). Add basic datastore delete test. Fix bug in create/delete: make sure extra information that was put into the data_dict is not returned from the logic function. --- ckanext/datastore/db.py | 4 ++- ckanext/datastore/logic/action/create.py | 5 ++- ckanext/datastore/logic/action/delete.py | 20 +++++------ ckanext/datastore/logic/auth/delete.py | 5 +-- ckanext/datastore/plugin.py | 8 +++-- ckanext/datastore/tests/test_datastore.py | 44 ++++++++++++++++++++++- 6 files changed, 69 insertions(+), 17 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index be077274987..1ab51c991fa 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -368,7 +368,9 @@ def delete(context, data_dict): data_dict['resource_id']) }) if not 'filter' in data_dict: - context['connection'].execute('drop table "{}"'.format(data_dict)) + context['connection'].execute( + 'drop table "{}"'.format(data_dict['resource_id']) + ) else: delete_data(context, data_dict) diff --git a/ckanext/datastore/logic/action/create.py b/ckanext/datastore/logic/action/create.py index 58ef8ce96dc..86f22245449 100644 --- a/ckanext/datastore/logic/action/create.py +++ b/ckanext/datastore/logic/action/create.py @@ -34,4 +34,7 @@ def datastore_create(context, data_dict): data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] - return db.create(context, data_dict) + result = db.create(context, data_dict) + result.pop('id') + result.pop('connection_url') + return result diff --git a/ckanext/datastore/logic/action/delete.py b/ckanext/datastore/logic/action/delete.py index 5243bf3b7f9..9abc0e4700b 100644 --- a/ckanext/datastore/logic/action/delete.py +++ b/ckanext/datastore/logic/action/delete.py @@ -3,8 +3,6 @@ import ckan.logic as logic import ckan.logic.action import ckan.lib.dictization -import ckan.plugins as p -import ckanext.datastore.logic.schema import ckanext.datastore.db as db log = logging.getLogger(__name__) @@ -15,23 +13,25 @@ def datastore_delete(context, data_dict): - '''Adds a new table to the datastore. + '''Deletes a table from the datastore. - :param resource_id: resource id that the data is going to be stored under. + :param resource_id: resource id that the data will be deleted from. :type resource_id: string - :param filter: Filter to do deleting on over eg {'name': 'fred'} if missing - delete whole table + :param filter: filter to do deleting on over (eg {'name': 'fred'}). + If missing delete whole table. :returns: original filters sent. :rtype: dictionary ''' - model = _get_or_bust(context, 'model') + _get_or_bust(context, 'model') + _get_or_bust(data_dict, 'resource_id') _check_access('datastore_delete', context, data_dict) - _get_or_bust(data_dict, 'resource_id') - data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] - return db.delete(context, data_dict) + result = db.delete(context, data_dict) + result.pop('id') + result.pop('connection_url') + return result diff --git a/ckanext/datastore/logic/auth/delete.py b/ckanext/datastore/logic/auth/delete.py index 2e3992cf873..ebd9a57e667 100644 --- a/ckanext/datastore/logic/auth/delete.py +++ b/ckanext/datastore/logic/auth/delete.py @@ -1,9 +1,9 @@ -import ckan.plugins as p import ckan.logic as logic from ckan.lib.base import _ _check_access = logic.check_access + def datastore_delete(context, data_dict): data_dict['id'] = data_dict.get('resource_id') user = context.get('user') @@ -12,6 +12,7 @@ def datastore_delete(context, data_dict): if not authorized: return {'success': False, - 'msg': _('User %s not authorized to read edit %s') % (str(user), data_dict['id'])} + 'msg': _('User %s not authorized to read edit %s') % \ + (str(user), data_dict['id'])} else: return {'success': True} diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index ab2fee05fb2..c6984c70dda 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -1,6 +1,8 @@ import ckan.plugins as p import ckanext.datastore.logic.action.create as action_create +import ckanext.datastore.logic.action.delete as action_delete import ckanext.datastore.logic.auth.create as auth_create +import ckanext.datastore.logic.auth.delete as auth_delete class DatastoreException(Exception): @@ -22,7 +24,9 @@ def configure(self, config): raise DatastoreException(error_msg) def get_actions(self): - return {'datastore_create': action_create.datastore_create} + return {'datastore_create': action_create.datastore_create, + 'datastore_delete': action_delete.datastore_delete} def get_auth_functions(self): - return {'datastore_create': auth_create.datastore_create} + return {'datastore_create': auth_create.datastore_create, + 'datastore_delete': auth_delete.datastore_delete} diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index bf5c7c7adf3..ae51e180660 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -6,7 +6,7 @@ import ckanext.datastore.db as db -class TestDatastore(tests.WsgiAppCase): +class TestDatastoreCreate(tests.WsgiAppCase): sysadmin_user = None normal_user = None @@ -323,3 +323,45 @@ def test_guess_types(self): res_dict = json.loads(res.body) assert res_dict['success'] is False + + +class TestDatastoreDelete(tests.WsgiAppCase): + sysadmin_user = None + normal_user = None + + @classmethod + def setup_class(cls): + p.load('datastore') + ctd.CreateTestData.create() + cls.sysadmin_user = model.User.get('testsysadmin') + cls.normal_user = model.User.get('annafan') + + @classmethod + def teardown_class(cls): + model.repo.rebuild_db() + + def setup(self): + resource = model.Package.get('annakarenina').resources[0] + self.data = { + 'resource_id': resource.id, + 'fields': [{'id': 'book', 'type': 'text'}, + {'id': 'author', 'type': 'text'}], + 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, + {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']}] + } + postparams = '%s=1' % json.dumps(self.data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + + def test_delete_basic(self): + data = {'resource_id': self.data['resource_id']} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_delete', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + assert res_dict['result'] == data From 7e31f903d23185ac3f34a2b50174e0a10db105eb Mon Sep 17 00:00:00 2001 From: John Glover Date: Mon, 30 Jul 2012 12:06:13 +0100 Subject: [PATCH 018/130] [#2733] Update basic delete test: make sure table has actually been deleted. --- ckanext/datastore/tests/test_datastore.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index ae51e180660..0b858a72bfe 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -1,4 +1,5 @@ import json +import sqlalchemy import ckan.plugins as p import ckan.lib.create_test_data as ctd import ckan.model as model @@ -357,7 +358,8 @@ def setup(self): assert res_dict['success'] is True def test_delete_basic(self): - data = {'resource_id': self.data['resource_id']} + id = self.data['resource_id'] + data = {'resource_id': id} postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_delete', params=postparams, @@ -365,3 +367,14 @@ def test_delete_basic(self): res_dict = json.loads(res.body) assert res_dict['success'] is True assert res_dict['result'] == data + + c = model.Session.connection() + + try: + # check that data was actually deleted: this should raise a + # ProgrammingError as the table should not exist any more + c.execute('select * from "{0}";'.format(id)) + raise Exception("Data not deleted") + except sqlalchemy.exc.ProgrammingError as e: + expected_msg = 'relation "{}" does not exist'.format(id) + assert expected_msg in str(e) From 81816553cc282c8b80de684843e572dff45f2e5c Mon Sep 17 00:00:00 2001 From: John Glover Date: Mon, 30 Jul 2012 14:16:05 +0100 Subject: [PATCH 019/130] [#2733] Check that resource ID exists for create/delete logic functions. Add test for deleting invalid resource ID. --- ckanext/datastore/logic/action/create.py | 18 ++++++++-------- ckanext/datastore/logic/action/delete.py | 17 +++++++-------- ckanext/datastore/tests/test_datastore.py | 25 ++++++++++++++++------- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/ckanext/datastore/logic/action/create.py b/ckanext/datastore/logic/action/create.py index 86f22245449..73272aaafa7 100644 --- a/ckanext/datastore/logic/action/create.py +++ b/ckanext/datastore/logic/action/create.py @@ -1,14 +1,10 @@ import logging import pylons import ckan.logic as logic -import ckan.logic.action -import ckan.lib.dictization +import ckan.plugins as p import ckanext.datastore.db as db log = logging.getLogger(__name__) - -_validate = ckan.lib.navl.dictization_functions.validate -_check_access = logic.check_access _get_or_bust = logic.get_or_bust @@ -26,11 +22,15 @@ def datastore_create(context, data_dict): :rtype: dictionary ''' - _get_or_bust(context, 'model') - _get_or_bust(data_dict, 'resource_id') - # TODO: check that resource_id exists in database + model = _get_or_bust(context, 'model') + id = _get_or_bust(data_dict, 'resource_id') + + if not model.Resource.get(id): + raise p.toolkit.ObjectNotFound(p.toolkit._( + 'Resource "{}" was not found.'.format(id) + )) - _check_access('datastore_create', context, data_dict) + p.toolkit.check_access('datastore_create', context, data_dict) data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] diff --git a/ckanext/datastore/logic/action/delete.py b/ckanext/datastore/logic/action/delete.py index 9abc0e4700b..25ec50887c0 100644 --- a/ckanext/datastore/logic/action/delete.py +++ b/ckanext/datastore/logic/action/delete.py @@ -1,14 +1,10 @@ import logging import pylons import ckan.logic as logic -import ckan.logic.action -import ckan.lib.dictization +import ckan.plugins as p import ckanext.datastore.db as db log = logging.getLogger(__name__) - -_validate = ckan.lib.navl.dictization_functions.validate -_check_access = logic.check_access _get_or_bust = logic.get_or_bust @@ -24,10 +20,15 @@ def datastore_delete(context, data_dict): :rtype: dictionary ''' - _get_or_bust(context, 'model') - _get_or_bust(data_dict, 'resource_id') + model = _get_or_bust(context, 'model') + id = _get_or_bust(data_dict, 'resource_id') + + if not model.Resource.get(id): + raise p.toolkit.ObjectNotFound(p.toolkit._( + 'Resource "{}" was not found.'.format(id) + )) - _check_access('datastore_delete', context, data_dict) + p.toolkit.check_access('datastore_delete', context, data_dict) data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index 0b858a72bfe..8c85b5ec819 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -336,20 +336,20 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') - - @classmethod - def teardown_class(cls): - model.repo.rebuild_db() - - def setup(self): resource = model.Package.get('annakarenina').resources[0] - self.data = { + cls.data = { 'resource_id': resource.id, 'fields': [{'id': 'book', 'type': 'text'}, {'id': 'author', 'type': 'text'}], 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']}] } + + @classmethod + def teardown_class(cls): + model.repo.rebuild_db() + + def _create(self): postparams = '%s=1' % json.dumps(self.data) auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_create', params=postparams, @@ -358,6 +358,7 @@ def setup(self): assert res_dict['success'] is True def test_delete_basic(self): + self._create() id = self.data['resource_id'] data = {'resource_id': id} postparams = '%s=1' % json.dumps(data) @@ -378,3 +379,13 @@ def test_delete_basic(self): except sqlalchemy.exc.ProgrammingError as e: expected_msg = 'relation "{}" does not exist'.format(id) assert expected_msg in str(e) + + model.Session.remove() + + def test_delete_invalid_resource_id(self): + postparams = '%s=1' % json.dumps({'resource_id': 'bad'}) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_delete', params=postparams, + extra_environ=auth, status=404) + res_dict = json.loads(res.body) + assert res_dict['success'] is False From e1c00b7b9bf02e4d9aa06a30fb77617a8b8d3684 Mon Sep 17 00:00:00 2001 From: John Glover Date: Mon, 30 Jul 2012 14:21:13 +0100 Subject: [PATCH 020/130] [#2733] Remove schema for now as not used. --- ckanext/datastore/logic/schema.py | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 ckanext/datastore/logic/schema.py diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py deleted file mode 100644 index 7c1f170e95e..00000000000 --- a/ckanext/datastore/logic/schema.py +++ /dev/null @@ -1,14 +0,0 @@ -from ckan.lib.navl.validators import (not_empty, - not_missing, - empty, - ignore) - - -def default_datastore_create_schema(): - # TODO: resource_id should have a resource_id_exists validator - return { - 'resource_id': [not_missing, not_empty, unicode], - 'fields': [ignore], - 'records': [ignore], - '__extras': [empty], - } From d8fb53323ce4ddf3a8a845c01e6f685aa6ba2d32 Mon Sep 17 00:00:00 2001 From: John Glover Date: Mon, 30 Jul 2012 15:21:45 +0100 Subject: [PATCH 021/130] [#2733] Test datastore delete filters. --- ckanext/datastore/db.py | 12 ++-- ckanext/datastore/tests/test_datastore.py | 74 ++++++++++++++++++++--- 2 files changed, 73 insertions(+), 13 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 1ab51c991fa..929680e7111 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -274,11 +274,11 @@ def insert_data(context, data_dict): def delete_data(context, data_dict): - filter = data_dict.get('filter') + filters = data_dict.get('filters') - if not isinstance(filter, dict): + if not isinstance(filters, dict): raise p.toolkit.ValidationError({ - 'filter': 'Not a json object'} + 'filters': 'Not a json object'} ) fields = _get_fields(context, data_dict) @@ -286,10 +286,10 @@ def delete_data(context, data_dict): where_clauses = [] values = [] - for field, value in filter.iteritems(): + for field, value in filters.iteritems(): if field not in field_ids: raise p.toolkit.ValidationError({ - 'filter': 'field "{}" not in table'} + 'filters': 'field "{}" not in table'} ) where_clauses.append('"{}" = %s'.format(field)) values.append(value) @@ -367,7 +367,7 @@ def delete(context, data_dict): 'resource_id': 'table for resource {0} does not exist'.format( data_dict['resource_id']) }) - if not 'filter' in data_dict: + if not 'filters' in data_dict: context['connection'].execute( 'drop table "{}"'.format(data_dict['resource_id']) ) diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index 8c85b5ec819..b9d9afad75d 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -342,7 +342,7 @@ def setup_class(cls): 'fields': [{'id': 'book', 'type': 'text'}, {'id': 'author', 'type': 'text'}], 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, - {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']}] + {'book': 'warandpeace', 'author': 'tolstoy'}] } @classmethod @@ -356,11 +356,10 @@ def _create(self): extra_environ=auth) res_dict = json.loads(res.body) assert res_dict['success'] is True + return res_dict - def test_delete_basic(self): - self._create() - id = self.data['resource_id'] - data = {'resource_id': id} + def _delete(self): + data = {'resource_id': self.data['resource_id']} postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_delete', params=postparams, @@ -368,16 +367,21 @@ def test_delete_basic(self): res_dict = json.loads(res.body) assert res_dict['success'] is True assert res_dict['result'] == data + return res_dict + def test_delete_basic(self): + self._create() + self._delete() + resource_id = self.data['resource_id'] c = model.Session.connection() try: # check that data was actually deleted: this should raise a # ProgrammingError as the table should not exist any more - c.execute('select * from "{0}";'.format(id)) + c.execute('select * from "{0}";'.format(resource_id)) raise Exception("Data not deleted") except sqlalchemy.exc.ProgrammingError as e: - expected_msg = 'relation "{}" does not exist'.format(id) + expected_msg = 'relation "{}" does not exist'.format(resource_id) assert expected_msg in str(e) model.Session.remove() @@ -389,3 +393,59 @@ def test_delete_invalid_resource_id(self): extra_environ=auth, status=404) res_dict = json.loads(res.body) assert res_dict['success'] is False + + def test_delete_filters(self): + self._create() + resource_id = self.data['resource_id'] + + # try and delete just the 'warandpeace' row + data = {'resource_id': resource_id, + 'filters': {'book': 'warandpeace'}} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_delete', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + + c = model.Session.connection() + result = c.execute('select * from "{0}";'.format(resource_id)) + results = [r for r in result] + assert len(results) == 1 + assert results[0].book == 'annakarenina' + model.Session.remove() + + # shouldn't delete anything + data = {'resource_id': resource_id, + 'filters': {'book': 'annakarenina', 'author': 'bad'}} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_delete', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + + c = model.Session.connection() + result = c.execute('select * from "{0}";'.format(resource_id)) + results = [r for r in result] + assert len(results) == 1 + assert results[0].book == 'annakarenina' + model.Session.remove() + + # delete the 'annakarenina' row + data = {'resource_id': resource_id, + 'filters': {'book': 'annakarenina', 'author': 'tolstoy'}} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_delete', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + + c = model.Session.connection() + result = c.execute('select * from "{0}";'.format(resource_id)) + results = [r for r in result] + assert len(results) == 0 + model.Session.remove() + + self._delete() From 99e53587423e4a972ba1c4253b5aaf6475b0bd2f Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 31 Jul 2012 09:37:26 +0100 Subject: [PATCH 022/130] [#2733] Add code skeleton for datastore_search. --- ckanext/datastore/db.py | 24 ++++++++++ ckanext/datastore/logic/action/get.py | 57 +++++++++++++++++++++++ ckanext/datastore/logic/auth/create.py | 9 ++-- ckanext/datastore/logic/auth/delete.py | 9 ++-- ckanext/datastore/logic/auth/get.py | 15 ++++++ ckanext/datastore/plugin.py | 8 +++- ckanext/datastore/tests/test_datastore.py | 39 ++++++++++++++++ 7 files changed, 147 insertions(+), 14 deletions(-) create mode 100644 ckanext/datastore/logic/action/get.py create mode 100644 ckanext/datastore/logic/auth/get.py diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 929680e7111..c87e1a2a555 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -305,6 +305,10 @@ def delete_data(context, data_dict): ) +def search_data(context, data_dict): + return data_dict + + def create(context, data_dict): ''' The first row will be used to guess types not in the fields and the @@ -381,3 +385,23 @@ def delete(context, data_dict): raise finally: context['connection'].close() + + +def search(context, data_dict): + engine = _get_engine(context, data_dict) + context['connection'] = engine.connect() + + try: + # check if table existes + result = context['connection'].execute( + 'select * from pg_tables where tablename = %s', + data_dict['resource_id'] + ).fetchone() + if not result: + raise p.toolkit.ValidationError({ + 'resource_id': 'table for resource {0} does not exist'.format( + data_dict['resource_id']) + }) + return search_data(context, data_dict) + finally: + context['connection'].close() diff --git a/ckanext/datastore/logic/action/get.py b/ckanext/datastore/logic/action/get.py new file mode 100644 index 00000000000..72ef2e5da2e --- /dev/null +++ b/ckanext/datastore/logic/action/get.py @@ -0,0 +1,57 @@ +import logging +import pylons +import ckan.logic as logic +import ckan.plugins as p +import ckanext.datastore.db as db + +log = logging.getLogger(__name__) +_get_or_bust = logic.get_or_bust + + +def datastore_search(context, data_dict): + '''Search a datastore table. + + :param resource_id: id of the data that is going to be selected. + :type resource_id: string + :param filters: matching conditions to select. + :type filters: dictionary + :param q: full text query + :type q: string + :param limit: maximum number of rows to return (default: 100) + :type limit: int + :param offset: offset the number of rows + :type offset: int + :param fields: ordered list of fields to return + (default: all fields in original order) + :type fields: list of strings + :param sort: comma separated field names with ordering + eg: "fieldname1, fieldname2 desc" + :type sort: string + + :returns: a dictionary containing the search parameters and the + search results. + keys: fields: same as datastore_create accepts + offset: query offset value + limit: query limit value + filters: query filters + total: number of total matching records + records: list of matching results + :rtype: dictionary + + ''' + model = _get_or_bust(context, 'model') + id = _get_or_bust(data_dict, 'resource_id') + + if not model.Resource.get(id): + raise p.toolkit.ObjectNotFound(p.toolkit._( + 'Resource "{}" was not found.'.format(id) + )) + + p.toolkit.check_access('datastore_search', context, data_dict) + + data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] + + result = db.search(context, data_dict) + result.pop('id') + result.pop('connection_url') + return result diff --git a/ckanext/datastore/logic/auth/create.py b/ckanext/datastore/logic/auth/create.py index f5c881a930c..095e358713a 100644 --- a/ckanext/datastore/logic/auth/create.py +++ b/ckanext/datastore/logic/auth/create.py @@ -1,18 +1,15 @@ -import ckan.logic as logic -from ckan.lib.base import _ - -_check_access = logic.check_access +import ckan.plugins as p def datastore_create(context, data_dict): data_dict['id'] = data_dict.get('resource_id') user = context.get('user') - authorized = _check_access('resource_update', context, data_dict) + authorized = p.toolkit.check_access('resource_update', context, data_dict) if not authorized: return {'success': False, - 'msg': _('User %s not authorized to read edit %s') % \ + 'msg': p.toolkit._('User %s not authorized to read edit %s') %\ (str(user), data_dict['id'])} else: return {'success': True} diff --git a/ckanext/datastore/logic/auth/delete.py b/ckanext/datastore/logic/auth/delete.py index ebd9a57e667..186fc24e4f0 100644 --- a/ckanext/datastore/logic/auth/delete.py +++ b/ckanext/datastore/logic/auth/delete.py @@ -1,18 +1,15 @@ -import ckan.logic as logic -from ckan.lib.base import _ - -_check_access = logic.check_access +import ckan.plugins as p def datastore_delete(context, data_dict): data_dict['id'] = data_dict.get('resource_id') user = context.get('user') - authorized = _check_access('resource_update', context, data_dict) + authorized = p.toolkit.check_access('resource_update', context, data_dict) if not authorized: return {'success': False, - 'msg': _('User %s not authorized to read edit %s') % \ + 'msg': p.toolkit._('User %s not authorized to read edit %s') %\ (str(user), data_dict['id'])} else: return {'success': True} diff --git a/ckanext/datastore/logic/auth/get.py b/ckanext/datastore/logic/auth/get.py new file mode 100644 index 00000000000..1db46b0597e --- /dev/null +++ b/ckanext/datastore/logic/auth/get.py @@ -0,0 +1,15 @@ +import ckan.plugins as p + + +def datastore_search(context, data_dict): + data_dict['id'] = data_dict.get('resource_id') + user = context.get('user') + + authorized = p.toolkit.check_access('resource_update', context, data_dict) + + if not authorized: + return {'success': False, + 'msg': p.toolkit._('User %s not authorized to read edit %s') %\ + (str(user), data_dict['id'])} + else: + return {'success': True} diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index c6984c70dda..a995c28ad72 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -1,8 +1,10 @@ import ckan.plugins as p import ckanext.datastore.logic.action.create as action_create import ckanext.datastore.logic.action.delete as action_delete +import ckanext.datastore.logic.action.get as action_get import ckanext.datastore.logic.auth.create as auth_create import ckanext.datastore.logic.auth.delete as auth_delete +import ckanext.datastore.logic.auth.get as auth_get class DatastoreException(Exception): @@ -25,8 +27,10 @@ def configure(self, config): def get_actions(self): return {'datastore_create': action_create.datastore_create, - 'datastore_delete': action_delete.datastore_delete} + 'datastore_delete': action_delete.datastore_delete, + 'datastore_search': action_get.datastore_search} def get_auth_functions(self): return {'datastore_create': auth_create.datastore_create, - 'datastore_delete': auth_delete.datastore_delete} + 'datastore_delete': auth_delete.datastore_delete, + 'datastore_search': auth_get.datastore_search} diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index b9d9afad75d..2f56eb89cd6 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -449,3 +449,42 @@ def test_delete_filters(self): model.Session.remove() self._delete() + + +class TestDatastoreSearch(tests.WsgiAppCase): + sysadmin_user = None + normal_user = None + + @classmethod + def setup_class(cls): + p.load('datastore') + ctd.CreateTestData.create() + cls.sysadmin_user = model.User.get('testsysadmin') + cls.normal_user = model.User.get('annafan') + resource = model.Package.get('annakarenina').resources[0] + cls.data = { + 'resource_id': resource.id, + 'fields': [{'id': 'book', 'type': 'text'}, + {'id': 'author', 'type': 'text'}], + 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, + {'book': 'warandpeace', 'author': 'tolstoy'}] + } + postparams = '%s=1' % json.dumps(cls.data) + auth = {'Authorization': str(cls.sysadmin_user.apikey)} + res = cls.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + + @classmethod + def teardown_class(cls): + model.repo.rebuild_db() + + def test_search_basic(self): + data = {'resource_id': self.data['resource_id']} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_search', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True From 50b636b3839c0101d69514a860536e2c75c08f60 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 31 Jul 2012 09:54:39 +0100 Subject: [PATCH 023/130] [#2733] Refactor: Move auth/action into single files, currently small functions with duplicated code. --- ckanext/datastore/logic/action.py | 119 +++++++++++++++++++++ ckanext/datastore/logic/action/__init__.py | 0 ckanext/datastore/logic/action/create.py | 40 ------- ckanext/datastore/logic/action/delete.py | 38 ------- ckanext/datastore/logic/action/get.py | 57 ---------- ckanext/datastore/logic/auth.py | 29 +++++ ckanext/datastore/logic/auth/__init__.py | 0 ckanext/datastore/logic/auth/create.py | 15 --- ckanext/datastore/logic/auth/delete.py | 15 --- ckanext/datastore/logic/auth/get.py | 15 --- ckanext/datastore/plugin.py | 20 ++-- 11 files changed, 156 insertions(+), 192 deletions(-) create mode 100644 ckanext/datastore/logic/action.py delete mode 100644 ckanext/datastore/logic/action/__init__.py delete mode 100644 ckanext/datastore/logic/action/create.py delete mode 100644 ckanext/datastore/logic/action/delete.py delete mode 100644 ckanext/datastore/logic/action/get.py create mode 100644 ckanext/datastore/logic/auth.py delete mode 100644 ckanext/datastore/logic/auth/__init__.py delete mode 100644 ckanext/datastore/logic/auth/create.py delete mode 100644 ckanext/datastore/logic/auth/delete.py delete mode 100644 ckanext/datastore/logic/auth/get.py diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py new file mode 100644 index 00000000000..e75d4e21323 --- /dev/null +++ b/ckanext/datastore/logic/action.py @@ -0,0 +1,119 @@ +import logging +import pylons +import ckan.logic as logic +import ckan.plugins as p +import ckanext.datastore.db as db + +log = logging.getLogger(__name__) +_get_or_bust = logic.get_or_bust + + +def datastore_create(context, data_dict): + '''Adds a new table to the datastore. + + :param resource_id: resource id that the data is going to be stored under. + :type resource_id: string + :param fields: fields/columns and their extra metadata. + :type fields: list of dictionaries + :param records: the data, eg: [{"dob": "2005", "some_stuff": ['a', b']}] + :type records: list of dictionaries + + :returns: the newly created data object. + :rtype: dictionary + + ''' + model = _get_or_bust(context, 'model') + id = _get_or_bust(data_dict, 'resource_id') + + if not model.Resource.get(id): + raise p.toolkit.ObjectNotFound(p.toolkit._( + 'Resource "{}" was not found.'.format(id) + )) + + p.toolkit.check_access('datastore_create', context, data_dict) + + data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] + + result = db.create(context, data_dict) + result.pop('id') + result.pop('connection_url') + return result + + +def datastore_delete(context, data_dict): + '''Deletes a table from the datastore. + + :param resource_id: resource id that the data will be deleted from. + :type resource_id: string + :param filter: filter to do deleting on over (eg {'name': 'fred'}). + If missing delete whole table. + + :returns: original filters sent. + :rtype: dictionary + + ''' + model = _get_or_bust(context, 'model') + id = _get_or_bust(data_dict, 'resource_id') + + if not model.Resource.get(id): + raise p.toolkit.ObjectNotFound(p.toolkit._( + 'Resource "{}" was not found.'.format(id) + )) + + p.toolkit.check_access('datastore_delete', context, data_dict) + + data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] + + result = db.delete(context, data_dict) + result.pop('id') + result.pop('connection_url') + return result + + +def datastore_search(context, data_dict): + '''Search a datastore table. + + :param resource_id: id of the data that is going to be selected. + :type resource_id: string + :param filters: matching conditions to select. + :type filters: dictionary + :param q: full text query + :type q: string + :param limit: maximum number of rows to return (default: 100) + :type limit: int + :param offset: offset the number of rows + :type offset: int + :param fields: ordered list of fields to return + (default: all fields in original order) + :type fields: list of strings + :param sort: comma separated field names with ordering + eg: "fieldname1, fieldname2 desc" + :type sort: string + + :returns: a dictionary containing the search parameters and the + search results. + keys: fields: same as datastore_create accepts + offset: query offset value + limit: query limit value + filters: query filters + total: number of total matching records + records: list of matching results + :rtype: dictionary + + ''' + model = _get_or_bust(context, 'model') + id = _get_or_bust(data_dict, 'resource_id') + + if not model.Resource.get(id): + raise p.toolkit.ObjectNotFound(p.toolkit._( + 'Resource "{}" was not found.'.format(id) + )) + + p.toolkit.check_access('datastore_search', context, data_dict) + + data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] + + result = db.search(context, data_dict) + result.pop('id') + result.pop('connection_url') + return result diff --git a/ckanext/datastore/logic/action/__init__.py b/ckanext/datastore/logic/action/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/ckanext/datastore/logic/action/create.py b/ckanext/datastore/logic/action/create.py deleted file mode 100644 index 73272aaafa7..00000000000 --- a/ckanext/datastore/logic/action/create.py +++ /dev/null @@ -1,40 +0,0 @@ -import logging -import pylons -import ckan.logic as logic -import ckan.plugins as p -import ckanext.datastore.db as db - -log = logging.getLogger(__name__) -_get_or_bust = logic.get_or_bust - - -def datastore_create(context, data_dict): - '''Adds a new table to the datastore. - - :param resource_id: resource id that the data is going to be stored under. - :type resource_id: string - :param fields: fields/columns and their extra metadata. - :type fields: list of dictionaries - :param records: the data, eg: [{"dob": "2005", "some_stuff": ['a', b']}] - :type records: list of dictionaries - - :returns: the newly created data object. - :rtype: dictionary - - ''' - model = _get_or_bust(context, 'model') - id = _get_or_bust(data_dict, 'resource_id') - - if not model.Resource.get(id): - raise p.toolkit.ObjectNotFound(p.toolkit._( - 'Resource "{}" was not found.'.format(id) - )) - - p.toolkit.check_access('datastore_create', context, data_dict) - - data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] - - result = db.create(context, data_dict) - result.pop('id') - result.pop('connection_url') - return result diff --git a/ckanext/datastore/logic/action/delete.py b/ckanext/datastore/logic/action/delete.py deleted file mode 100644 index 25ec50887c0..00000000000 --- a/ckanext/datastore/logic/action/delete.py +++ /dev/null @@ -1,38 +0,0 @@ -import logging -import pylons -import ckan.logic as logic -import ckan.plugins as p -import ckanext.datastore.db as db - -log = logging.getLogger(__name__) -_get_or_bust = logic.get_or_bust - - -def datastore_delete(context, data_dict): - '''Deletes a table from the datastore. - - :param resource_id: resource id that the data will be deleted from. - :type resource_id: string - :param filter: filter to do deleting on over (eg {'name': 'fred'}). - If missing delete whole table. - - :returns: original filters sent. - :rtype: dictionary - - ''' - model = _get_or_bust(context, 'model') - id = _get_or_bust(data_dict, 'resource_id') - - if not model.Resource.get(id): - raise p.toolkit.ObjectNotFound(p.toolkit._( - 'Resource "{}" was not found.'.format(id) - )) - - p.toolkit.check_access('datastore_delete', context, data_dict) - - data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] - - result = db.delete(context, data_dict) - result.pop('id') - result.pop('connection_url') - return result diff --git a/ckanext/datastore/logic/action/get.py b/ckanext/datastore/logic/action/get.py deleted file mode 100644 index 72ef2e5da2e..00000000000 --- a/ckanext/datastore/logic/action/get.py +++ /dev/null @@ -1,57 +0,0 @@ -import logging -import pylons -import ckan.logic as logic -import ckan.plugins as p -import ckanext.datastore.db as db - -log = logging.getLogger(__name__) -_get_or_bust = logic.get_or_bust - - -def datastore_search(context, data_dict): - '''Search a datastore table. - - :param resource_id: id of the data that is going to be selected. - :type resource_id: string - :param filters: matching conditions to select. - :type filters: dictionary - :param q: full text query - :type q: string - :param limit: maximum number of rows to return (default: 100) - :type limit: int - :param offset: offset the number of rows - :type offset: int - :param fields: ordered list of fields to return - (default: all fields in original order) - :type fields: list of strings - :param sort: comma separated field names with ordering - eg: "fieldname1, fieldname2 desc" - :type sort: string - - :returns: a dictionary containing the search parameters and the - search results. - keys: fields: same as datastore_create accepts - offset: query offset value - limit: query limit value - filters: query filters - total: number of total matching records - records: list of matching results - :rtype: dictionary - - ''' - model = _get_or_bust(context, 'model') - id = _get_or_bust(data_dict, 'resource_id') - - if not model.Resource.get(id): - raise p.toolkit.ObjectNotFound(p.toolkit._( - 'Resource "{}" was not found.'.format(id) - )) - - p.toolkit.check_access('datastore_search', context, data_dict) - - data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] - - result = db.search(context, data_dict) - result.pop('id') - result.pop('connection_url') - return result diff --git a/ckanext/datastore/logic/auth.py b/ckanext/datastore/logic/auth.py new file mode 100644 index 00000000000..88ec645cd69 --- /dev/null +++ b/ckanext/datastore/logic/auth.py @@ -0,0 +1,29 @@ +import ckan.plugins as p + + +def _datastore_auth(context, data_dict): + data_dict['id'] = data_dict.get('resource_id') + user = context.get('user') + + authorized = p.toolkit.check_access('resource_update', context, data_dict) + + if not authorized: + return { + 'success': False, + 'msg': p.toolkit._('User {} not authorized to update resource {}'\ + .format(str(user), data_dict['id'])) + } + else: + return {'success': True} + + +def datastore_create(context, data_dict): + return _datastore_auth(context, data_dict) + + +def datastore_delete(context, data_dict): + return _datastore_auth(context, data_dict) + + +def datastore_search(context, data_dict): + return _datastore_auth(context, data_dict) diff --git a/ckanext/datastore/logic/auth/__init__.py b/ckanext/datastore/logic/auth/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/ckanext/datastore/logic/auth/create.py b/ckanext/datastore/logic/auth/create.py deleted file mode 100644 index 095e358713a..00000000000 --- a/ckanext/datastore/logic/auth/create.py +++ /dev/null @@ -1,15 +0,0 @@ -import ckan.plugins as p - - -def datastore_create(context, data_dict): - data_dict['id'] = data_dict.get('resource_id') - user = context.get('user') - - authorized = p.toolkit.check_access('resource_update', context, data_dict) - - if not authorized: - return {'success': False, - 'msg': p.toolkit._('User %s not authorized to read edit %s') %\ - (str(user), data_dict['id'])} - else: - return {'success': True} diff --git a/ckanext/datastore/logic/auth/delete.py b/ckanext/datastore/logic/auth/delete.py deleted file mode 100644 index 186fc24e4f0..00000000000 --- a/ckanext/datastore/logic/auth/delete.py +++ /dev/null @@ -1,15 +0,0 @@ -import ckan.plugins as p - - -def datastore_delete(context, data_dict): - data_dict['id'] = data_dict.get('resource_id') - user = context.get('user') - - authorized = p.toolkit.check_access('resource_update', context, data_dict) - - if not authorized: - return {'success': False, - 'msg': p.toolkit._('User %s not authorized to read edit %s') %\ - (str(user), data_dict['id'])} - else: - return {'success': True} diff --git a/ckanext/datastore/logic/auth/get.py b/ckanext/datastore/logic/auth/get.py deleted file mode 100644 index 1db46b0597e..00000000000 --- a/ckanext/datastore/logic/auth/get.py +++ /dev/null @@ -1,15 +0,0 @@ -import ckan.plugins as p - - -def datastore_search(context, data_dict): - data_dict['id'] = data_dict.get('resource_id') - user = context.get('user') - - authorized = p.toolkit.check_access('resource_update', context, data_dict) - - if not authorized: - return {'success': False, - 'msg': p.toolkit._('User %s not authorized to read edit %s') %\ - (str(user), data_dict['id'])} - else: - return {'success': True} diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index a995c28ad72..849ba69c906 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -1,10 +1,6 @@ import ckan.plugins as p -import ckanext.datastore.logic.action.create as action_create -import ckanext.datastore.logic.action.delete as action_delete -import ckanext.datastore.logic.action.get as action_get -import ckanext.datastore.logic.auth.create as auth_create -import ckanext.datastore.logic.auth.delete as auth_delete -import ckanext.datastore.logic.auth.get as auth_get +import ckanext.datastore.logic.action as action +import ckanext.datastore.logic.auth as auth class DatastoreException(Exception): @@ -26,11 +22,11 @@ def configure(self, config): raise DatastoreException(error_msg) def get_actions(self): - return {'datastore_create': action_create.datastore_create, - 'datastore_delete': action_delete.datastore_delete, - 'datastore_search': action_get.datastore_search} + return {'datastore_create': action.datastore_create, + 'datastore_delete': action.datastore_delete, + 'datastore_search': action.datastore_search} def get_auth_functions(self): - return {'datastore_create': auth_create.datastore_create, - 'datastore_delete': auth_delete.datastore_delete, - 'datastore_search': auth_get.datastore_search} + return {'datastore_create': auth.datastore_create, + 'datastore_delete': auth.datastore_delete, + 'datastore_search': auth.datastore_search} From 68574937d3023367f22607b9c3e4b35e30db4ae2 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 31 Jul 2012 14:36:20 +0100 Subject: [PATCH 024/130] [#2733] Add basic search (just processing filters and fields parameters). --- ckanext/datastore/db.py | 53 ++++++++++++++++++++--- ckanext/datastore/logic/action.py | 2 +- ckanext/datastore/tests/test_datastore.py | 41 ++++++++++++++++++ 3 files changed, 89 insertions(+), 7 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index c87e1a2a555..c17e5ee9a87 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -273,19 +273,18 @@ def insert_data(context, data_dict): context['connection'].execute(sql_string, rows) -def delete_data(context, data_dict): - filters = data_dict.get('filters') +def _where_from_filters(field_ids, data_dict): + 'Return a SQL WHERE clause from data_dict filters' + filters = data_dict.get('filters', {}) if not isinstance(filters, dict): raise p.toolkit.ValidationError({ 'filters': 'Not a json object'} ) - fields = _get_fields(context, data_dict) - field_ids = set([field['id'] for field in fields]) - where_clauses = [] values = [] + for field, value in filters.iteritems(): if field not in field_ids: raise p.toolkit.ValidationError({ @@ -295,17 +294,59 @@ def delete_data(context, data_dict): values.append(value) where_clause = ' and '.join(where_clauses) + return where_clause, values + + +def delete_data(context, data_dict): + fields = _get_fields(context, data_dict) + field_ids = set([field['id'] for field in fields]) + where_clause, where_values = _where_from_filters(field_ids, data_dict) context['connection'].execute( 'delete from "{}" where {}'.format( data_dict['resource_id'], where_clause ), - values + where_values ) def search_data(context, data_dict): + all_fields = _get_fields(context, data_dict) + all_field_ids = set([field['id'] for field in all_fields]) + + fields = data_dict.get('fields') + + if fields: + check_fields(context, fields) + field_ids = set([field['id'] for field in fields]) + + for field in field_ids: + if not field in all_field_ids: + raise p.toolkit.ValidationError({ + 'fields': 'field "{}" not in table'.format(field)} + ) + else: + fields = all_fields + field_ids = all_field_ids + + select_columns = ', '.join(field_ids) + + where_clause, where_values = _where_from_filters(all_field_ids, data_dict) + + sql_string = 'select {} from "{}"'.format( + select_columns, data_dict['resource_id'] + ) + + if where_clause: + sql_string += ' where {}'.format(where_clause) + + results = context['connection'].execute(sql_string, where_values) + + data_dict['total'] = results.rowcount + records = [(dict((f, r[f]) for f in field_ids)) for r in results] + data_dict['records'] = records + return data_dict diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index e75d4e21323..aa856b7a10c 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -85,7 +85,7 @@ def datastore_search(context, data_dict): :type offset: int :param fields: ordered list of fields to return (default: all fields in original order) - :type fields: list of strings + :type fields: list of dictionaries :param sort: comma separated field names with ordering eg: "fieldname1, fieldname2 desc" :type sort: string diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index 2f56eb89cd6..c49d3f970ea 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -488,3 +488,44 @@ def test_search_basic(self): extra_environ=auth) res_dict = json.loads(res.body) assert res_dict['success'] is True + result = res_dict['result'] + assert result['total'] == len(self.data['records']) + assert result['records'] == self.data['records'] + + def test_search_invalid_field(self): + data = {'resource_id': self.data['resource_id'], + 'fields': [{'id': 'bad'}]} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_search', params=postparams, + extra_environ=auth, status=409) + res_dict = json.loads(res.body) + assert res_dict['success'] is False + + def test_search_fields(self): + data = {'resource_id': self.data['resource_id'], + 'fields': [{'id': 'book'}]} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_search', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + result = res_dict['result'] + assert result['total'] == len(self.data['records']) + assert result['records'] == [{'book': 'annakarenina'}, + {'book': 'warandpeace'}] + + def test_search_filters(self): + data = {'resource_id': self.data['resource_id'], + 'filters': {'book': 'annakarenina'}} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_search', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + result = res_dict['result'] + assert result['total'] == 1 + assert result['records'] == [{'book': 'annakarenina', + 'author': 'tolstoy'}] From 23447062f6c4d0e88ffdbb8665d0e85b1a392678 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 31 Jul 2012 15:28:31 +0100 Subject: [PATCH 025/130] [#2733] Implement datastore_search offset and limit. --- ckanext/datastore/db.py | 25 +++++++++++-------- ckanext/datastore/tests/test_datastore.py | 29 +++++++++++++++++++++++ 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index c17e5ee9a87..ed7249fe8f3 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -294,6 +294,8 @@ def _where_from_filters(field_ids, data_dict): values.append(value) where_clause = ' and '.join(where_clauses) + if where_clause: + where_clause = 'where ' + where_clause return where_clause, values @@ -303,7 +305,7 @@ def delete_data(context, data_dict): where_clause, where_values = _where_from_filters(field_ids, data_dict) context['connection'].execute( - 'delete from "{}" where {}'.format( + 'delete from "{}" {}'.format( data_dict['resource_id'], where_clause ), @@ -331,19 +333,22 @@ def search_data(context, data_dict): field_ids = all_field_ids select_columns = ', '.join(field_ids) - where_clause, where_values = _where_from_filters(all_field_ids, data_dict) + limit = data_dict.get('limit', 100) + offset = data_dict.get('offset', 0) - sql_string = 'select {} from "{}"'.format( - select_columns, data_dict['resource_id'] - ) - - if where_clause: - sql_string += ' where {}'.format(where_clause) - + sql_string = '''select {}, count(*) over() as full_count + from "{}" {} limit {} offset {}'''\ + .format(select_columns, data_dict['resource_id'], where_clause, + limit, offset) results = context['connection'].execute(sql_string, where_values) + results = [r for r in results] + + if results: + data_dict['total'] = results[0]['full_count'] + else: + data_dict['total'] = 0 - data_dict['total'] = results.rowcount records = [(dict((f, r[f]) for f in field_ids)) for r in results] data_dict['records'] = records diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index c49d3f970ea..c0077eb6abc 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -529,3 +529,32 @@ def test_search_filters(self): assert result['total'] == 1 assert result['records'] == [{'book': 'annakarenina', 'author': 'tolstoy'}] + + def test_search_limit(self): + data = {'resource_id': self.data['resource_id'], + 'limit': 1} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_search', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + result = res_dict['result'] + assert result['total'] == 2 + assert result['records'] == [{'book': 'annakarenina', + 'author': 'tolstoy'}] + + def test_search_offset(self): + data = {'resource_id': self.data['resource_id'], + 'limit': 1, + 'offset': 1} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_search', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + result = res_dict['result'] + assert result['total'] == 2 + assert result['records'] == [{'book': 'warandpeace', + 'author': 'tolstoy'}] From e3918114010e60c88fe0884c2572b688cfe6dd1d Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 31 Jul 2012 16:32:34 +0100 Subject: [PATCH 026/130] [#2733] Implement full text search in datastore_search ('q'). --- ckanext/datastore/db.py | 12 +++++++---- ckanext/datastore/tests/test_datastore.py | 26 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index ed7249fe8f3..3dec899a1ee 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -273,8 +273,8 @@ def insert_data(context, data_dict): context['connection'].execute(sql_string, rows) -def _where_from_filters(field_ids, data_dict): - 'Return a SQL WHERE clause from data_dict filters' +def _where(field_ids, data_dict): + 'Return a SQL WHERE clause from data_dict filters and q' filters = data_dict.get('filters', {}) if not isinstance(filters, dict): @@ -293,6 +293,10 @@ def _where_from_filters(field_ids, data_dict): where_clauses.append('"{}" = %s'.format(field)) values.append(value) + q = data_dict.get('q') + if q: + where_clauses.append('_full_text @@ to_tsquery(\'{}\')'.format(q)) + where_clause = ' and '.join(where_clauses) if where_clause: where_clause = 'where ' + where_clause @@ -302,7 +306,7 @@ def _where_from_filters(field_ids, data_dict): def delete_data(context, data_dict): fields = _get_fields(context, data_dict) field_ids = set([field['id'] for field in fields]) - where_clause, where_values = _where_from_filters(field_ids, data_dict) + where_clause, where_values = _where(field_ids, data_dict) context['connection'].execute( 'delete from "{}" {}'.format( @@ -333,7 +337,7 @@ def search_data(context, data_dict): field_ids = all_field_ids select_columns = ', '.join(field_ids) - where_clause, where_values = _where_from_filters(all_field_ids, data_dict) + where_clause, where_values = _where(all_field_ids, data_dict) limit = data_dict.get('limit', 100) offset = data_dict.get('offset', 0) diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index c0077eb6abc..a4474e35b82 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -558,3 +558,29 @@ def test_search_offset(self): assert result['total'] == 2 assert result['records'] == [{'book': 'warandpeace', 'author': 'tolstoy'}] + + def test_search_full_text(self): + data = {'resource_id': self.data['resource_id'], + 'q': 'annakarenina'} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_search', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + result = res_dict['result'] + assert result['total'] == 1 + assert result['records'] == [{'book': 'annakarenina', + 'author': 'tolstoy'}] + + data = {'resource_id': self.data['resource_id'], + 'q': 'tolstoy'} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_search', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + result = res_dict['result'] + assert result['total'] == 2 + assert result['records'] == self.data['records'] From 67e6f4c385a414d19192a93f6fd8553593886cba Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 31 Jul 2012 17:14:12 +0100 Subject: [PATCH 027/130] [#2733] Implement datastore_search sort parameter. --- ckanext/datastore/db.py | 9 ++++-- ckanext/datastore/tests/test_datastore.py | 35 ++++++++++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 3dec899a1ee..207bb7a45e0 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -341,10 +341,15 @@ def search_data(context, data_dict): limit = data_dict.get('limit', 100) offset = data_dict.get('offset', 0) + if data_dict.get('sort'): + sort = 'order by {}'.format(data_dict['sort']) + else: + sort = '' + sql_string = '''select {}, count(*) over() as full_count - from "{}" {} limit {} offset {}'''\ + from "{}" {} {} limit {} offset {}'''\ .format(select_columns, data_dict['resource_id'], where_clause, - limit, offset) + sort, limit, offset) results = context['connection'].execute(sql_string, where_values) results = [r for r in results] diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index a4474e35b82..6f4d3fa3d07 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -530,6 +530,40 @@ def test_search_filters(self): assert result['records'] == [{'book': 'annakarenina', 'author': 'tolstoy'}] + def test_search_sort(self): + data = {'resource_id': self.data['resource_id'], + 'sort': 'book asc'} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_search', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + result = res_dict['result'] + assert result['total'] == 2 + + expected_records = [ + {'book': 'annakarenina', 'author': 'tolstoy'}, + {'book': 'warandpeace', 'author': 'tolstoy'} + ] + assert result['records'] == expected_records + + data = {'resource_id': self.data['resource_id'], + 'sort': 'book desc'} + postparams = '%s=1' % json.dumps(data) + res = self.app.post('/api/action/datastore_search', params=postparams, + extra_environ=auth) + res_dict = json.loads(res.body) + assert res_dict['success'] is True + result = res_dict['result'] + assert result['total'] == 2 + + expected_records = [ + {'book': 'warandpeace', 'author': 'tolstoy'}, + {'book': 'annakarenina', 'author': 'tolstoy'} + ] + assert result['records'] == expected_records + def test_search_limit(self): data = {'resource_id': self.data['resource_id'], 'limit': 1} @@ -576,7 +610,6 @@ def test_search_full_text(self): data = {'resource_id': self.data['resource_id'], 'q': 'tolstoy'} postparams = '%s=1' % json.dumps(data) - auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_search', params=postparams, extra_environ=auth) res_dict = json.loads(res.body) From b264dcbc84a912b0476dba3f126c88209ab3b428 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 31 Jul 2012 17:31:39 +0100 Subject: [PATCH 028/130] [#2733] Check that limit and offset are integers. --- ckanext/datastore/db.py | 12 ++++++++++++ ckanext/datastore/tests/test_datastore.py | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 207bb7a45e0..bcf84df8d77 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -23,6 +23,15 @@ def _is_valid_field_name(name): return True +def _validate_int(i, field_name): + try: + int(i) + except ValueError: + raise p.toolkit.ValidationError({ + 'field_name': '{} is not an integer'.format(i) + }) + + def _get_engine(context, data_dict): 'Get either read or write engine.' connection_url = data_dict['connection_url'] @@ -341,6 +350,9 @@ def search_data(context, data_dict): limit = data_dict.get('limit', 100) offset = data_dict.get('offset', 0) + _validate_int(limit, 'limit') + _validate_int(offset, 'offset') + if data_dict.get('sort'): sort = 'order by {}'.format(data_dict['sort']) else: diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index 6f4d3fa3d07..40136f8fc15 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -578,6 +578,16 @@ def test_search_limit(self): assert result['records'] == [{'book': 'annakarenina', 'author': 'tolstoy'}] + def test_search_invalid_limit(self): + data = {'resource_id': self.data['resource_id'], + 'limit': 'bad'} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_search', params=postparams, + extra_environ=auth, status=409) + res_dict = json.loads(res.body) + assert res_dict['success'] is False + def test_search_offset(self): data = {'resource_id': self.data['resource_id'], 'limit': 1, @@ -593,6 +603,16 @@ def test_search_offset(self): assert result['records'] == [{'book': 'warandpeace', 'author': 'tolstoy'}] + def test_search_invalid_offset(self): + data = {'resource_id': self.data['resource_id'], + 'offset': 'bad'} + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_search', params=postparams, + extra_environ=auth, status=409) + res_dict = json.loads(res.body) + assert res_dict['success'] is False + def test_search_full_text(self): data = {'resource_id': self.data['resource_id'], 'q': 'annakarenina'} From 697ed53a35f7a177c08203dd5f6b45e3c3414ae4 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 31 Jul 2012 17:41:01 +0100 Subject: [PATCH 029/130] [#2733] Bug fix: pass 'q' as %s instead of inserting value into string. --- ckanext/datastore/db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index bcf84df8d77..4c8e17ec351 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -304,7 +304,8 @@ def _where(field_ids, data_dict): q = data_dict.get('q') if q: - where_clauses.append('_full_text @@ to_tsquery(\'{}\')'.format(q)) + where_clauses.append('_full_text @@ to_tsquery(%s)'.format(q)) + values.append(q) where_clause = ' and '.join(where_clauses) if where_clause: From abd25e415cd49be39d2814ef3924fff127b12621 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 31 Jul 2012 17:56:28 +0100 Subject: [PATCH 030/130] [#2733] Tidy up: remove unneeded 'factor' call. --- ckanext/datastore/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 4c8e17ec351..6e218729365 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -304,7 +304,7 @@ def _where(field_ids, data_dict): q = data_dict.get('q') if q: - where_clauses.append('_full_text @@ to_tsquery(%s)'.format(q)) + where_clauses.append('_full_text @@ to_tsquery(%s)') values.append(q) where_clause = ' and '.join(where_clauses) From 90239263877217f742042c5ddf2ccbd5beba29dc Mon Sep 17 00:00:00 2001 From: tobes Date: Mon, 6 Aug 2012 18:34:59 +0100 Subject: [PATCH 031/130] [#2618] Debug status now sets fanstatic response --- ckan/config/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index 16496bc0a97..ecd2065406e 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -69,7 +69,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf): #app = QueueLogMiddleware(app) # Fanstatic - if config.get('ckan.include_support', '').lower()[:3] == 'dev': + if asbool(config.get('debug', False)): fanstatic_config = { 'versioning' : True, 'recompute_hashes' : True, From 61984f932e1a4b5088733c688945d78b57412d44 Mon Sep 17 00:00:00 2001 From: tobes Date: Mon, 6 Aug 2012 18:35:55 +0100 Subject: [PATCH 032/130] [#2618] Fanstatic auto setup changes --- ckan/html_resources/__init__.py | 64 +++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index cb3abf7f65a..97ae7b676b9 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -121,7 +121,7 @@ def __init__(self, library, relpath, if argument is None: continue elif isinstance(argument, basestring): - mode_resource = Resource(library, argument, bottom=bottom, renderer=renderer) + mode_resource = Resource(library, argument, bottom=bottom, renderer=renderer, dont_bundle=dont_bundle) else: # The dependencies of a mode resource should be the same # or a subset of the dependencies this mode replaces. @@ -172,7 +172,7 @@ def min_path(path): if f.endswith('.css'): return path[:-4] + '.min.css' - def minify(filename, min_function): + def minify(filename, resource_path, min_function): ''' Minify file path using min_function. ''' # if the minified file was modified after the source file we can # assume that it is up-to-date @@ -187,10 +187,10 @@ def minify(filename, min_function): f.close() print 'minified %s' % path - def create_resource(filename, path, filepath): + def create_resource(path, lib_name): ''' create the fanstatic Resource ''' # resource_name is name of the file without the .js/.css - resource_name = '.'.join(filename.split('.')[:-1]) + rel_path, filename = os.path.split(path) kw = {} path_min = min_path(os.path.join(resource_path, filename)) if os.path.exists(path_min): @@ -200,12 +200,12 @@ def create_resource(filename, path, filepath): renderer = core.render_js if filename.endswith('.css'): renderer = core.render_css - if resource_name in depends: + if path in depends: dependencies = [] - for dependency in depends[resource_name]: - dependencies.append(getattr(module, dependency)) + for dependency in depends[path]: + dependencies.append(getattr(module, '%s/%s' % (lib_name, dependency))) kw['depends'] = dependencies - if resource_name in dont_bundle: + if path in dont_bundle: kw['dont_bundle'] = True # FIXME needs config.ini options enabled if False: @@ -215,11 +215,12 @@ def create_resource(filename, path, filepath): condition=condition, renderer=renderer, other_browsers=other_browsers) - + filename = os.path.join(rel_path, filename) resource = Resource(library, filename, **kw) # add the resource to this module - fanstatic_name = '%s/%s' % (filepath, resource_name) + fanstatic_name = '%s/%s' % (lib_name, filename) setattr(module, fanstatic_name, resource) + return resource order = [] dont_bundle = [] @@ -248,27 +249,35 @@ def create_resource(filename, path, filepath): module = sys.modules[__name__] # process each .js/.css file found + file_list = [] for dirname, dirnames, filenames in os.walk(resource_path): - for x in reversed(order): - if x in filenames: - filenames.remove(x) - filenames.insert(0, x) for f in filenames: + rel_path = dirname[len(path):] + if rel_path: + rel_path = rel_path[1:] + filepath = os.path.join(rel_path, f) if f.endswith('.js') and not f.endswith('.min.js'): - minify(f, jsmin) - create_resource(f, resource_path, path) + minify(f, dirname, jsmin) + file_list.append(filepath) if f.endswith('.css') and not f.endswith('.min.css'): - minify(f, cssmin) - create_resource(f, resource_path, path) + minify(f, dirname, cssmin) + file_list.append(filepath) + + for x in reversed(order): + if x in file_list: + file_list.remove(x) + file_list.insert(0, x) + for f in file_list: + create_resource(f, name) # add groups for group_name in groups: members = [] for member in groups[group_name]: - fanstatic_name = '%s/%s' % (path, member) + fanstatic_name = '%s/%s' % (name, member) members.append(getattr(module, fanstatic_name)) group = Group(members) - fanstatic_name = '%s/%s' % (path, group_name) + fanstatic_name = '%s/%s' % (name, group_name) setattr(module, fanstatic_name, group) # finally add the library to this module setattr(module, name, library) @@ -276,10 +285,13 @@ def create_resource(filename, path, filepath): registry = get_library_registry() registry.add(library) +base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'public', 'base', 'javascript')) +create_library('base', base_path) + -# create our libraries here from any subdirectories -for dirname, dirnames, filenames in os.walk(os.path.dirname(__file__)): - if dirname == os.path.dirname(__file__): - continue - lib_name = os.path.basename(dirname) - create_library(lib_name, lib_name) +### create our libraries here from any subdirectories +##for dirname, dirnames, filenames in os.walk(os.path.dirname(__file__)): +## if dirname == os.path.dirname(__file__): +## continue +## lib_name = os.path.basename(dirname) +## create_library(lib_name, lib_name) From 9a0a27254b4e718cd6a006890c45103a73e7779b Mon Sep 17 00:00:00 2001 From: tobes Date: Mon, 6 Aug 2012 18:37:48 +0100 Subject: [PATCH 033/130] [#2618] Use fanstatic --- ckan/templates/snippets/scripts.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ckan/templates/snippets/scripts.html b/ckan/templates/snippets/scripts.html index 2e32fe5b4d6..c28eb846482 100644 --- a/ckan/templates/snippets/scripts.html +++ b/ckan/templates/snippets/scripts.html @@ -6,7 +6,7 @@ - +{# @@ -26,4 +26,5 @@ - + #} +{% resource 'base/main' %} From e54665fa9bc7c71add89520e87bf10a4c8da9dff Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 7 Aug 2012 10:39:58 +0100 Subject: [PATCH 034/130] [#2375] Minor pep8 fix --- ckan/controllers/home.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/controllers/home.py b/ckan/controllers/home.py index 7a69075e9df..e2eea51a79d 100644 --- a/ckan/controllers/home.py +++ b/ckan/controllers/home.py @@ -141,7 +141,7 @@ def _form_to_db_schema(group_type=None): # We get all the packages or at least too many so # limit it to just 2 group_dict['packages'] = group_dict['packages'][:2] - return {'group_dict' :group_dict} + return {'group_dict': group_dict} global dirty_cached_group_stuff if not dirty_cached_group_stuff: From a995360a33eb9815c561e2ed35816ff07d5c9080 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 7 Aug 2012 10:40:54 +0100 Subject: [PATCH 035/130] [#2618] Bundle patches for fanstatic --- ckan/html_resources/__init__.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index 97ae7b676b9..3da2c7c57ea 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -157,8 +157,33 @@ def __init__(self, library, relpath, self.library.register(self) core.Resource.__init__ = __init__ -# Fanstatic Patch # +def render(self, library_url): + + + paths = [resource.relpath for resource in self._resources] + # URL may become too long: + # http://www.boutell.com/newfaq/misc/urllength.html + relpath = ''.join([core.BUNDLE_PREFIX, ';'.join(paths)]) + + return self.renderer('%s/%s' % (library_url, relpath)) + +core.Bundle.render = render +def fits(self, resource): + if resource.dont_bundle: + return False + # an empty resource fits anything + if not self._resources: + return True + # a resource fits if it's like the resources already inside + bundle_resource = self._resources[0] + return (resource.library is bundle_resource.library and + resource.renderer is bundle_resource.renderer and + (resource.ext == '.js' or + resource.dirname == bundle_resource.dirname)) + +core.Bundle.fits = fits +# Fanstatic Patch # def create_library(name, path): ''' Creates a fanstatic library `name` with the contents of a From 43ef93bbf4dbb8c027bc0358f5396113e707b0a9 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 7 Aug 2012 11:25:19 +0100 Subject: [PATCH 036/130] [#2618] Allow resource order to be forced --- ckan/html_resources/__init__.py | 38 ++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index 3da2c7c57ea..3c7090d91c8 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -67,7 +67,8 @@ def __init__(self, library, relpath, renderer=None, debug=None, dont_bundle=False, - minified=None): + minified=None, + order=0): self.library = library fullpath = os.path.normpath(os.path.join(library.path, relpath)) if core._resource_file_existence_checking and not os.path.exists(fullpath): @@ -79,6 +80,7 @@ def __init__(self, library, relpath, self.dirname += '/' self.bottom = bottom self.dont_bundle = dont_bundle + self.forced_order = order self.ext = os.path.splitext(self.relpath)[1] if renderer is None: @@ -183,6 +185,40 @@ def fits(self, resource): resource.dirname == bundle_resource.dirname)) core.Bundle.fits = fits + +def sort_resources(resources): + """Sort resources for inclusion on web page. + + A number of rules are followed: + + * resources are always grouped per renderer (.js, .css, etc) + * resources that depend on other resources are sorted later + * resources are grouped by library, if the dependencies allow it + * libraries are sorted by name, if dependencies allow it + * resources are sorted by resource path if they both would be + sorted the same otherwise. + + The only purpose of sorting on library is so we can + group resources per library, so that bundles can later be created + of them if bundling support is enabled. + + Note this sorting algorithm guarantees a consistent ordering, no + matter in what order resources were needed. + """ + for resource in resources: + resource.library.init_library_nr() + + def key(resource): + return ( + resource.order, + resource.forced_order, + resource.library.library_nr, + resource.library.name, + resource.dependency_nr, + resource.relpath) + return sorted(resources, key=key) + +core.sort_resources = sort_resources # Fanstatic Patch # def create_library(name, path): From ebb0524517dd0f33ad8247d2372261f0853d39bb Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 7 Aug 2012 11:25:57 +0100 Subject: [PATCH 037/130] [#2618] Set resource order on lib creation --- ckan/html_resources/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index 3c7090d91c8..c5cf50f0d60 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -248,7 +248,7 @@ def minify(filename, resource_path, min_function): f.close() print 'minified %s' % path - def create_resource(path, lib_name): + def create_resource(path, lib_name, count): ''' create the fanstatic Resource ''' # resource_name is name of the file without the .js/.css rel_path, filename = os.path.split(path) @@ -268,6 +268,7 @@ def create_resource(path, lib_name): kw['depends'] = dependencies if path in dont_bundle: kw['dont_bundle'] = True + kw['order'] = count # FIXME needs config.ini options enabled if False: other_browsers = False @@ -328,8 +329,10 @@ def create_resource(path, lib_name): if x in file_list: file_list.remove(x) file_list.insert(0, x) + count = 0 for f in file_list: - create_resource(f, name) + create_resource(f, name, count) + count += 1 # add groups for group_name in groups: From 1259a518f688bc2ce1c1c0dbdf959e4ca49de615 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 7 Aug 2012 13:33:06 +0100 Subject: [PATCH 038/130] [#2618] Better naming in fanstatic patch --- ckan/html_resources/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index c5cf50f0d60..05b6ef4f505 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -68,7 +68,7 @@ def __init__(self, library, relpath, debug=None, dont_bundle=False, minified=None, - order=0): + custom_order=0): self.library = library fullpath = os.path.normpath(os.path.join(library.path, relpath)) if core._resource_file_existence_checking and not os.path.exists(fullpath): @@ -80,7 +80,7 @@ def __init__(self, library, relpath, self.dirname += '/' self.bottom = bottom self.dont_bundle = dont_bundle - self.forced_order = order + self.custom_order = custom_order self.ext = os.path.splitext(self.relpath)[1] if renderer is None: @@ -211,7 +211,7 @@ def sort_resources(resources): def key(resource): return ( resource.order, - resource.forced_order, + resource.custom_order, resource.library.library_nr, resource.library.name, resource.dependency_nr, From 26f43360ab66972709a55b5fbd1c3d910d3660ae Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 7 Aug 2012 13:42:34 +0100 Subject: [PATCH 039/130] [#2618] Custom render order added to Fanstatic Resource --- ckan/html_resources/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index 05b6ef4f505..deda49ad425 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -68,6 +68,7 @@ def __init__(self, library, relpath, debug=None, dont_bundle=False, minified=None, + custom_renderer_order=None, custom_order=0): self.library = library fullpath = os.path.normpath(os.path.join(library.path, relpath)) @@ -99,6 +100,8 @@ def __init__(self, library, relpath, self.order, _ = core.inclusion_renderers.get( self.ext, (sys.maxint, None)) + if custom_renderer_order: + self.order = custom_renderer_order assert not isinstance(depends, basestring) self.depends = set() if depends is not None: From 81025bfca9b24db2287ce7d020742d9214ae5ba8 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 7 Aug 2012 14:09:28 +0100 Subject: [PATCH 040/130] [#2618] Better passing of vars in fanstatic --- ckan/html_resources/__init__.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index deda49ad425..dc7caed56a6 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -60,16 +60,20 @@ def __call__(self, url): # Fanstatic Patch # # FIXME add full license info & push upstream -def __init__(self, library, relpath, - depends=None, - supersedes=None, - bottom=False, - renderer=None, - debug=None, - dont_bundle=False, - minified=None, - custom_renderer_order=None, - custom_order=0): +def __init__(self, library, relpath, **kw): + + depends = kw.get('depends', None) + supersedes = kw.get('supersedes', None) + bottom = kw.get('bottom', False) + renderer = kw.get('renderer', None) + dont_bundle = kw.get('dont_bundle', False) + custom_renderer_order = kw.get('custom_renderer_order', None) + custom_order = kw.get('custom_order', 0) + + # we don't want to pass these again + minified = kw.pop('minified', None) + debug = kw.pop('debug', None) + self.library = library fullpath = os.path.normpath(os.path.join(library.path, relpath)) if core._resource_file_existence_checking and not os.path.exists(fullpath): @@ -126,7 +130,7 @@ def __init__(self, library, relpath, if argument is None: continue elif isinstance(argument, basestring): - mode_resource = Resource(library, argument, bottom=bottom, renderer=renderer, dont_bundle=dont_bundle) + mode_resource = Resource(library, argument, **kw) else: # The dependencies of a mode resource should be the same # or a subset of the dependencies this mode replaces. From bd96d4d8968b6055e0f28b13d091338c54f4b5e5 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 7 Aug 2012 14:10:36 +0100 Subject: [PATCH 041/130] [#2618] Use correct param when creating resource --- ckan/html_resources/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index dc7caed56a6..e36ce9bc720 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -275,7 +275,7 @@ def create_resource(path, lib_name, count): kw['depends'] = dependencies if path in dont_bundle: kw['dont_bundle'] = True - kw['order'] = count + kw['custom_order'] = count # FIXME needs config.ini options enabled if False: other_browsers = False From acfe3b2690e45068e4f2c7c52c846c852494a543 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 7 Aug 2012 14:11:14 +0100 Subject: [PATCH 042/130] [#2618] Get minified files in sub dirs --- ckan/html_resources/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index e36ce9bc720..65e1bbfe4ba 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -260,6 +260,7 @@ def create_resource(path, lib_name, count): # resource_name is name of the file without the .js/.css rel_path, filename = os.path.split(path) kw = {} + filename = os.path.join(rel_path, filename) path_min = min_path(os.path.join(resource_path, filename)) if os.path.exists(path_min): kw['minified'] = min_path(filename) @@ -284,7 +285,6 @@ def create_resource(path, lib_name, count): condition=condition, renderer=renderer, other_browsers=other_browsers) - filename = os.path.join(rel_path, filename) resource = Resource(library, filename, **kw) # add the resource to this module fanstatic_name = '%s/%s' % (lib_name, filename) From f411b79d9eecb734461405722068451371e74394 Mon Sep 17 00:00:00 2001 From: kindly Date: Tue, 7 Aug 2012 16:32:34 +0100 Subject: [PATCH 043/130] [#2733] parse and validate sort --- ckanext/datastore/db.py | 47 ++++++++++++++++++++--- ckanext/datastore/tests/test_datastore.py | 4 +- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 4c8e17ec351..f38911f9bab 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -2,6 +2,7 @@ import ckan.plugins as p import json import datetime +import shlex _pg_types = {} _type_names = set() @@ -304,7 +305,7 @@ def _where(field_ids, data_dict): q = data_dict.get('q') if q: - where_clauses.append('_full_text @@ to_tsquery(%s)'.format(q)) + where_clauses.append('_full_text @@ to_tsquery(%s)') values.append(q) where_clause = ' and '.join(where_clauses) @@ -312,6 +313,45 @@ def _where(field_ids, data_dict): where_clause = 'where ' + where_clause return where_clause, values +def _sort(context, sort, field_ids): + + if not sort: + return '' + + if isinstance(sort, basestring): + clauses = sort.split(',') + elif isinstance(sort, list): + clauses = sort + else: + raise p.toolkit.ValidationError({ + 'sort': 'sort is not a list or a string' + }) + + clause_parsed = [] + + for clause in clauses: + clause_parts = shlex.split(clause) + if len(clause_parts) == 1: + field, sort = clause_parts[0], 'asc' + elif len(clause_parts) == 2: + field, sort = clause_parts + else: + raise p.toolkit.ValidationError({ + 'sort': 'not valid syntax for sort clause' + }) + if field not in field_ids: + raise p.toolkit.ValidationError({ + 'sort': 'field {} not it table'.format(field) + }) + if sort.lower() not in ('asc', 'desc'): + raise p.toolkit.ValidationError({ + 'sort': 'sorting can only be asc or desc' + }) + clause_parsed.append('"{}" {}'.format(field, sort)) + + if clause_parsed: + return "order by " + ", ".join(clause_parsed) + def delete_data(context, data_dict): fields = _get_fields(context, data_dict) @@ -354,10 +394,7 @@ def search_data(context, data_dict): _validate_int(limit, 'limit') _validate_int(offset, 'offset') - if data_dict.get('sort'): - sort = 'order by {}'.format(data_dict['sort']) - else: - sort = '' + sort = _sort(context, data_dict.get('sort'), field_ids) sql_string = '''select {}, count(*) over() as full_count from "{}" {} {} limit {} offset {}'''\ diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index 40136f8fc15..85b1d961857 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -532,7 +532,7 @@ def test_search_filters(self): def test_search_sort(self): data = {'resource_id': self.data['resource_id'], - 'sort': 'book asc'} + 'sort': 'book asc, author desc'} postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_search', params=postparams, @@ -549,7 +549,7 @@ def test_search_sort(self): assert result['records'] == expected_records data = {'resource_id': self.data['resource_id'], - 'sort': 'book desc'} + 'sort': ['book desc', '"author" asc']} postparams = '%s=1' % json.dumps(data) res = self.app.post('/api/action/datastore_search', params=postparams, extra_environ=auth) From 2fc489dc9913babb7d9d04a08c7f4c9c216850b9 Mon Sep 17 00:00:00 2001 From: kindly Date: Tue, 7 Aug 2012 20:03:25 +0100 Subject: [PATCH 044/130] [#2733] make search permission more liberal --- ckanext/datastore/logic/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datastore/logic/auth.py b/ckanext/datastore/logic/auth.py index 88ec645cd69..0cac0f18f31 100644 --- a/ckanext/datastore/logic/auth.py +++ b/ckanext/datastore/logic/auth.py @@ -26,4 +26,4 @@ def datastore_delete(context, data_dict): def datastore_search(context, data_dict): - return _datastore_auth(context, data_dict) + return {'success': True} From aedb12450977a102be479611044af184ee98e2b9 Mon Sep 17 00:00:00 2001 From: kindly Date: Tue, 7 Aug 2012 23:09:14 +0100 Subject: [PATCH 045/130] [#2733] fix issue with auth change --- ckanext/datastore/logic/action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index aa856b7a10c..2be78a60522 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -114,6 +114,6 @@ def datastore_search(context, data_dict): data_dict['connection_url'] = pylons.config['ckan.datastore_write_url'] result = db.search(context, data_dict) - result.pop('id') + result.pop('id', None) result.pop('connection_url') return result From 97782d9d30dd93e9b1dcf05066375f0e9e15c9d1 Mon Sep 17 00:00:00 2001 From: kindly Date: Wed, 8 Aug 2012 22:10:10 +0100 Subject: [PATCH 046/130] [#2733] add json type --- ckanext/datastore/db.py | 81 ++++++++++++++++------- ckanext/datastore/tests/test_datastore.py | 80 +++++++++++----------- 2 files changed, 96 insertions(+), 65 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index f38911f9bab..d6088e30616 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1,5 +1,6 @@ import sqlalchemy import ckan.plugins as p +import psycopg2.extras import json import datetime import shlex @@ -53,6 +54,15 @@ def _cache_types(context): for result in results: _pg_types[result[0]] = result[1] _type_names.add(result[1]) + if '_json' not in _type_names: + connection.execute('create type "_json" as (json text, extra text)') + _pg_types.clear() + ## redo cache types with json now availiable. + return _cache_types(context) + + psycopg2.extras.register_composite('_json', connection.connection, + True) + def _get_type(context, oid): @@ -64,6 +74,8 @@ def _guess_type(field): 'Simple guess type of field, only allowed are integer, numeric and text' data_types = set([int, float]) + if isinstance(field, (dict, list)): + return '_json' for data_type in list(data_types): try: data_type(field) @@ -109,14 +121,13 @@ def json_get_values(obj, current_list=None): for item in obj: json_get_values(item, current_list) if isinstance(obj, dict): - for item in dict.values(): + for item in obj.values(): json_get_values(item, current_list) return current_list def check_fields(context, fields): 'Check if field types are valid.' - _cache_types(context) for field in fields: if field.get('type') and not field['type'] in _type_names: raise p.toolkit.ValidationError({ @@ -127,6 +138,16 @@ def check_fields(context, fields): 'fields': '{0} is not a valid field name'.format(field['id']) }) +def convert(data, type): + if data is None: + return None + if type == '_json': + return json.loads(data[0]) + if isinstance(data, datetime.datetime): + return data.isoformat() + if isinstance(data, (int, float)): + return data + return unicode(data) def create_table(context, data_dict): 'Create table from combination of fields and first row of data.' @@ -237,11 +258,11 @@ def insert_data(context, data_dict): return fields = _get_fields(context, data_dict) - field_names = [field['id'] for field in fields] + ['_full_text'] - sql_columns = ", ".join(['"%s"' % name for name in field_names]) + field_names = [field['id'] for field in fields] + sql_columns = ", ".join(['"%s"' % name for name in field_names] + + ['_full_text']) rows = [] - ## clean up and validate data for num, record in enumerate(data_dict['records']): @@ -255,7 +276,7 @@ def insert_data(context, data_dict): if extra_keys: raise p.toolkit.ValidationError({ 'records': 'row {} has extra keys "{}"'.format( - num, + num + 1, ', '.join(list(extra_keys)) ) }) @@ -264,9 +285,10 @@ def insert_data(context, data_dict): row = [] for field in fields: value = record.get(field['id']) - if isinstance(value, (dict, list)): + if field['type'].lower() == '_json' and value: full_text.extend(json_get_values(value)) - value = json.dumps(value) + ## a tuple with an empty second value + value = (json.dumps(value), '') elif field['type'].lower() == 'text' and value: full_text.append(value) row.append(value) @@ -277,7 +299,7 @@ def insert_data(context, data_dict): sql_string = 'insert into "{0}" ({1}) values ({2});'.format( data_dict['resource_id'], sql_columns, - ', '.join(['%s' for field in field_names]) + ', '.join(['%s' for field in field_names + ['_full_text']]) ) context['connection'].execute(sql_string, rows) @@ -369,13 +391,13 @@ def delete_data(context, data_dict): def search_data(context, data_dict): all_fields = _get_fields(context, data_dict) - all_field_ids = set([field['id'] for field in all_fields]) + all_field_ids = [field['id'] for field in all_fields] + all_field_ids.insert(0,'_id') fields = data_dict.get('fields') if fields: - check_fields(context, fields) - field_ids = set([field['id'] for field in fields]) + field_ids = fields for field in field_ids: if not field in all_field_ids: @@ -383,7 +405,6 @@ def search_data(context, data_dict): 'fields': 'field "{}" not in table'.format(field)} ) else: - fields = all_fields field_ids = all_field_ids select_columns = ', '.join(field_ids) @@ -396,24 +417,35 @@ def search_data(context, data_dict): sort = _sort(context, data_dict.get('sort'), field_ids) - sql_string = '''select {}, count(*) over() as full_count + sql_string = '''select {}, count(*) over() as "_full_count" from "{}" {} {} limit {} offset {}'''\ .format(select_columns, data_dict['resource_id'], where_clause, sort, limit, offset) results = context['connection'].execute(sql_string, where_values) - results = [r for r in results] - - if results: - data_dict['total'] = results[0]['full_count'] - else: - data_dict['total'] = 0 - records = [(dict((f, r[f]) for f in field_ids)) for r in results] + result_fields = [] + for field in results.cursor.description: + result_fields.append({ + 'id': field[0], + 'type': _get_type(context, field[1]) + }) + result_fields.pop() # remove _full_count + + data_dict['total'] = 0 + + records = [] + for row in results: + converted_row = {} + if not data_dict['total']: + data_dict['total'] = row['_full_count'] + for field in result_fields: + converted_row[field['id']] = convert(row[field['id']], + field['type']) + records.append(converted_row) data_dict['records'] = records - + data_dict['fields'] = result_fields return data_dict - def create(context, data_dict): ''' The first row will be used to guess types not in the fields and the @@ -437,6 +469,7 @@ def create(context, data_dict): ''' engine = _get_engine(context, data_dict) context['connection'] = engine.connect() + _cache_types(context) # close connection at all cost. try: @@ -463,6 +496,7 @@ def create(context, data_dict): def delete(context, data_dict): engine = _get_engine(context, data_dict) context['connection'] = engine.connect() + _cache_types(context) try: # check if table existes @@ -495,6 +529,7 @@ def delete(context, data_dict): def search(context, data_dict): engine = _get_engine(context, data_dict) context['connection'] = engine.connect() + _cache_types(context) try: # check if table existes diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index 85b1d961857..1cb6105a5dd 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -134,9 +134,10 @@ def test_create_basic(self): data = { 'resource_id': resource.id, 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'author', 'type': 'text'}], - 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, + {'id': 'author', 'type': '_json'}], + 'records': [ {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']}, + {'book': 'annakarenina', 'author': ['tolstoy', 'putin']}, {'book': 'warandpeace'}] # treat author as null } postparams = '%s=1' % json.dumps(data) @@ -156,8 +157,7 @@ def test_create_basic(self): assert results.rowcount == 3 for i, row in enumerate(results): assert data['records'][i].get('book') == row['book'] - assert (data['records'][i].get('author') == row['author'] - or data['records'][i].get('author') == json.loads(row['author'])) + assert data['records'][i].get('author') == (json.loads(row['author'][0]) if row['author'] else None) results = c.execute('''select * from "{0}" where _full_text @@ 'warandpeace' '''.format(resource.id)) assert results.rowcount == 1 @@ -169,7 +169,7 @@ def test_create_basic(self): ####### insert again simple data2 = { 'resource_id': resource.id, - 'records': [{'book': 'hagji murat', 'author': 'tolstoy'}] + 'records': [{'book': 'hagji murat', 'author': ['tolstoy']}] } postparams = '%s=1' % json.dumps(data2) @@ -188,8 +188,7 @@ def test_create_basic(self): all_data = data['records'] + data2['records'] for i, row in enumerate(results): assert all_data[i].get('book') == row['book'] - assert (all_data[i].get('author') == row['author'] - or all_data[i].get('author') == json.loads(row['author'])) + assert all_data[i].get('author') == (json.loads(row['author'][0]) if row['author'] else None) results = c.execute('''select * from "{0}" where _full_text @@ 'tolstoy' '''.format(resource.id)) assert results.rowcount == 3 @@ -199,7 +198,7 @@ def test_create_basic(self): data3 = { 'resource_id': resource.id, 'records': [{'book': 'crime and punsihment', - 'author': 'dostoevsky', 'rating': 'good'}] + 'author': ['dostoevsky'], 'rating': 'good'}] } postparams = '%s=1' % json.dumps(data3) @@ -216,11 +215,9 @@ def test_create_basic(self): assert results.rowcount == 5 all_data = data['records'] + data2['records'] + data3['records'] - print all_data for i, row in enumerate(results): assert all_data[i].get('book') == row['book'], (i, all_data[i].get('book'), row['book']) - assert (all_data[i].get('author') == row['author'] - or all_data[i].get('author') == json.loads(row['author'])) + assert all_data[i].get('author') == (json.loads(row['author'][0]) if row['author'] else None) results = c.execute('''select * from "{0}" where _full_text @@ 'dostoevsky' '''.format(resource.id)) assert results.rowcount == 2 @@ -230,7 +227,7 @@ def test_guess_types(self): resource = model.Package.get('annakarenina').resources[1] data = { 'resource_id': resource.id, - 'fields': [{'id': 'author', 'type': 'text'}, + 'fields': [{'id': 'author', 'type': '_json'}, {'id': 'count'}, {'id': 'book'}, {'id': 'date'}], @@ -250,14 +247,13 @@ def test_guess_types(self): types = [db._pg_types[field[1]] for field in results.cursor.description] - assert types == [u'int4', u'tsvector', u'text', u'int4', + assert types == [u'int4', u'tsvector', u'_json', u'int4', u'text', u'timestamp', u'int4'], types assert results.rowcount == 3 for i, row in enumerate(results): assert data['records'][i].get('book') == row['book'] - assert (data['records'][i].get('author') == row['author'] - or data['records'][i].get('author') == json.loads(row['author'])) + assert data['records'][i].get('author') == (json.loads(row['author'][0]) if row['author'] else None) model.Session.remove() ### extend types @@ -274,7 +270,7 @@ def test_guess_types(self): ], 'records': [{'book': 'annakarenina', 'author': 'tolstoy', 'count': 1, 'date': '2005-12-01', 'count2': 2, - 'count3': 432, 'date2': '2005-12-01'}] + 'nested': [1,2], 'date2': '2005-12-01'}] } postparams = '%s=1' % json.dumps(data) @@ -290,14 +286,14 @@ def test_guess_types(self): assert types == [u'int4', # id u'tsvector', # fulltext - u'text', # author + u'_json', # author u'int4', # count u'text', # book u'timestamp', # date u'int4', # count2 u'text', # extra u'timestamp', # date2 - u'int4', # count3 + u'_json', # count3 ], types ### fields resupplied in wrong order @@ -466,8 +462,9 @@ def setup_class(cls): 'resource_id': resource.id, 'fields': [{'id': 'book', 'type': 'text'}, {'id': 'author', 'type': 'text'}], - 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, - {'book': 'warandpeace', 'author': 'tolstoy'}] + 'records': [{'book': 'annakarenina', 'author': 'tolstoy', 'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]}, + {'book': 'warandpeace', 'author': 'tolstoy', 'nested': {'a':'b'}} + ] } postparams = '%s=1' % json.dumps(cls.data) auth = {'Authorization': str(cls.sysadmin_user.apikey)} @@ -476,6 +473,14 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True + cls.expected_records = [{u'published': u'2005-03-01T00:00:00', + u'_id': 1, + u'nested': [u'b', {u'moo': u'moo'}], u'book': u'annakarenina', u'author': u'tolstoy'}, + {u'published': None, + u'_id': 2, + u'nested': {u'a': u'b'}, u'book': u'warandpeace', u'author': u'tolstoy'}] + + @classmethod def teardown_class(cls): model.repo.rebuild_db() @@ -490,7 +495,7 @@ def test_search_basic(self): assert res_dict['success'] is True result = res_dict['result'] assert result['total'] == len(self.data['records']) - assert result['records'] == self.data['records'] + assert result['records'] == self.expected_records def test_search_invalid_field(self): data = {'resource_id': self.data['resource_id'], @@ -504,7 +509,7 @@ def test_search_invalid_field(self): def test_search_fields(self): data = {'resource_id': self.data['resource_id'], - 'fields': [{'id': 'book'}]} + 'fields': ['book']} postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_search', params=postparams, @@ -514,7 +519,7 @@ def test_search_fields(self): result = res_dict['result'] assert result['total'] == len(self.data['records']) assert result['records'] == [{'book': 'annakarenina'}, - {'book': 'warandpeace'}] + {'book': 'warandpeace'}], result['records'] def test_search_filters(self): data = {'resource_id': self.data['resource_id'], @@ -527,8 +532,7 @@ def test_search_filters(self): assert res_dict['success'] is True result = res_dict['result'] assert result['total'] == 1 - assert result['records'] == [{'book': 'annakarenina', - 'author': 'tolstoy'}] + assert result['records'] == [self.expected_records[0]] def test_search_sort(self): data = {'resource_id': self.data['resource_id'], @@ -542,11 +546,7 @@ def test_search_sort(self): result = res_dict['result'] assert result['total'] == 2 - expected_records = [ - {'book': 'annakarenina', 'author': 'tolstoy'}, - {'book': 'warandpeace', 'author': 'tolstoy'} - ] - assert result['records'] == expected_records + assert result['records'] == self.expected_records, result['records'] data = {'resource_id': self.data['resource_id'], 'sort': ['book desc', '"author" asc']} @@ -558,11 +558,7 @@ def test_search_sort(self): result = res_dict['result'] assert result['total'] == 2 - expected_records = [ - {'book': 'warandpeace', 'author': 'tolstoy'}, - {'book': 'annakarenina', 'author': 'tolstoy'} - ] - assert result['records'] == expected_records + assert result['records'] == self.expected_records[::-1] def test_search_limit(self): data = {'resource_id': self.data['resource_id'], @@ -575,8 +571,7 @@ def test_search_limit(self): assert res_dict['success'] is True result = res_dict['result'] assert result['total'] == 2 - assert result['records'] == [{'book': 'annakarenina', - 'author': 'tolstoy'}] + assert result['records'] == [self.expected_records[0]] def test_search_invalid_limit(self): data = {'resource_id': self.data['resource_id'], @@ -600,8 +595,7 @@ def test_search_offset(self): assert res_dict['success'] is True result = res_dict['result'] assert result['total'] == 2 - assert result['records'] == [{'book': 'warandpeace', - 'author': 'tolstoy'}] + assert result['records'] == [self.expected_records[1]] def test_search_invalid_offset(self): data = {'resource_id': self.data['resource_id'], @@ -624,8 +618,7 @@ def test_search_full_text(self): assert res_dict['success'] is True result = res_dict['result'] assert result['total'] == 1 - assert result['records'] == [{'book': 'annakarenina', - 'author': 'tolstoy'}] + assert result['records'] == [self.expected_records[0]] data = {'resource_id': self.data['resource_id'], 'q': 'tolstoy'} @@ -636,4 +629,7 @@ def test_search_full_text(self): assert res_dict['success'] is True result = res_dict['result'] assert result['total'] == 2 - assert result['records'] == self.data['records'] + assert result['records'] == self.expected_records, result['records'] + + assert result['fields'] == [{u'type': u'int4', u'id': u'_id'}, {u'type': u'text', u'id': u'book'}, {u'type': u'text', u'id': u'author'}, {u'type': u'timestamp', u'id': u'published'}, {u'type': u'_json', u'id': u'nested'}], result['fields'] + From e5d95b7b47160545af1ea0aebac16327447954b4 Mon Sep 17 00:00:00 2001 From: kindly Date: Thu, 9 Aug 2012 02:44:54 +0100 Subject: [PATCH 047/130] [#2733] add extra date types, make sure side effect free queries --- ckanext/datastore/db.py | 23 ++++++++++++++++------- ckanext/datastore/logic/action.py | 1 + 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index d6088e30616..dd1d8fc4155 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -9,10 +9,15 @@ _type_names = set() _engines = {} -_iso_formats = ['%Y-%m-%d', +_date_formats = ['%Y-%m-%d', '%Y-%m-%d %H:%M:%S', - '%Y-%m-%dT%H:%M:%S'] - + '%Y-%m-%dT%H:%M:%S', + '%Y-%m-%dT%H:%M:%SZ', + '%d/%m/%Y', + '%m/%d/%Y', + '%d-%m-%Y', + '%m-%d-%Y', + ] def _is_valid_field_name(name): ''' @@ -73,9 +78,12 @@ def _get_type(context, oid): def _guess_type(field): 'Simple guess type of field, only allowed are integer, numeric and text' data_types = set([int, float]) - if isinstance(field, (dict, list)): return '_json' + if isinstance(field, int): + return 'int' + if isinstance(field, float): + return 'float' for data_type in list(data_types): try: data_type(field) @@ -89,7 +97,7 @@ def _guess_type(field): return 'numeric' ##try iso dates - for format in _iso_formats: + for format in _date_formats: try: datetime.datetime.strptime(field, format) return 'timestamp' @@ -259,7 +267,7 @@ def insert_data(context, data_dict): fields = _get_fields(context, data_dict) field_names = [field['id'] for field in fields] - sql_columns = ", ".join(['"%s"' % name for name in field_names] + sql_columns = ", ".join(['"%s"' % name for name in field_names] + ['_full_text']) rows = [] @@ -407,7 +415,8 @@ def search_data(context, data_dict): else: field_ids = all_field_ids - select_columns = ', '.join(field_ids) + select_columns = ', '.join(['"{}"'.format(field_id) + for field_id in field_ids]) where_clause, where_values = _where(all_field_ids, data_dict) limit = data_dict.get('limit', 100) offset = data_dict.get('offset', 0) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 2be78a60522..489a954e8ec 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -70,6 +70,7 @@ def datastore_delete(context, data_dict): return result +@logic.side_effect_free def datastore_search(context, data_dict): '''Search a datastore table. From 5501b1350e9cd92b6d956ff0f2e71df9e1702f65 Mon Sep 17 00:00:00 2001 From: Daniel Lewis Date: Fri, 10 Aug 2012 09:57:39 +0100 Subject: [PATCH 048/130] modified: doc/datastore.rst Changed datastore.rst to detail most recent datastore api changes --- doc/datastore.rst | 77 +++++++++++------------------------------------ 1 file changed, 17 insertions(+), 60 deletions(-) diff --git a/doc/datastore.rst b/doc/datastore.rst index 9f3de6a919c..04a940d8cd5 100644 --- a/doc/datastore.rst +++ b/doc/datastore.rst @@ -33,80 +33,42 @@ queries over the spreadsheet contents. The DataStore Data API ====================== -The DataStore's Data API, which derives from the underlying ElasticSearch -data-table, is RESTful and JSON-based with extensive query capabilities. +The DataStore's Data API, which derives from the underlying data-table, is RESTful and JSON-based with extensive query capabilities. Each resource in a CKAN instance has an associated DataStore 'table'. This table will be accessible via a web interface at:: /api/data/{resource-id} -This interface to this data is *exactly* the same as that provided by -ElasticSearch to documents of a specific type in one of its indices. - For a detailed tutorial on using this API see :doc:`using-data-api`. Installation and Configuration ============================== -The DataStore uses ElasticSearch_ as the persistence and query layer with CKAN -wrapping this with a thin authorization and authentication layer. - -It also requires the use of Nginx as your webserver as its XSendfile_ feature -is used to transparently hand off data requests to ElasticSeach internally. - -.. _ElasticSearch: http://www.elasticsearch.org/ -.. _XSendfile: http://wiki.nginx.org/XSendfile - -1. Install ElasticSearch_ -------------------------- - -Please see the ElasticSearch_ documentation. - -2. Configure Nginx ------------------- +The DataStore in previous lives required a custom setup of ElasticSearch and Nginx, but that is no more, as it can use any relational database management system (PostgreSQL for example). -As previously mentioned, Nginx will be used on top of CKAN to forward -requests to Elastic Search. CKAN will still be served by Apache or the -development server (Paster), but all requests will be forwarded to it -by Ngnix. - -This is an example of an Nginx configuration file. Note the two locations -defined, `/` will point to the server running CKAN (Apache or Paster), and -`/elastic/` to the Elastic Search instance:: +To enable datastore features in CKAN +------------------------------------ - server { - listen 80 default; - server_name localhost; +In your config file... - access_log /var/log/nginx/localhost.access.log; +Enable the ckan datastore:: - location / { - # location of apache or ckan under paster - proxy_pass http://127.0.0.1:8080; - proxy_set_header Host $host; - } - location /elastic/ { - internal; - # location of elastic search - proxy_pass http://:127.0.0.1:9200/; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - } + ckan.datastore.enabled = 1 + +Ensure that the datastore extension is enabled:: -.. note:: update the proxy_pass field value to point to your ElasticSearch - instance (if it is not localhost and default port). + ckan.plugins = datastore + +Ensure that the ckan.datastore_write_url variable is set:: -Remember that after setting up Nginx, you need to access CKAN via its port -(80), not the Apache or Paster (5000) one, otherwise the DataStore won't work. + ckan.datastore_write_url = postgresql://ckanuser:pass@localhost/ckantest + +To test you can create a new datastore, so on linux command line do:: -3. Enable datastore features in CKAN ------------------------------------- + curl -X POST http://127.0.0.1:5000/api/3/action/datastore_create -H "Authorization: {YOUR-API-KEY}" -d "{\"resource_id\": \"{PRE-EXISTING-RESOURCE-ID}\", \"fields\": [ {\"id\": \"a\"}, {\"id\": \"b\"} ], \"records\": [ { \"a\": 1, \"b\": \"xyz\"}, {\"a\": 2, \"b\": \"zzz\"} ]}" -In your config file set:: - ckan.datastore.enabled = 1 .. _datastorer: @@ -130,11 +92,6 @@ How It Works (Technically) 1. Request arrives at e.g. /dataset/{id}/resource/{resource-id}/data 2. CKAN checks authentication and authorization. -3. (Assuming OK) CKAN hands (internally) to ElasticSearch which handles the +3. (Assuming OK) CKAN hands (internally) to the database querying system which handles the request - * To do this we use Nginx's Sendfile / Accel-Redirect feature. This allows - us to hand off a user request *directly* to ElasticSearch after the - authentication and authorization. This avoids the need to proxy the - request and results through CKAN code. - From fdec7089b49a3dcd1859e11222427609b4d7a418 Mon Sep 17 00:00:00 2001 From: Daniel Lewis Date: Fri, 10 Aug 2012 12:45:48 +0100 Subject: [PATCH 049/130] Modified datastore.rst and changed using-data-api.rst to reflect new datastore api --- doc/datastore.rst | 21 +- doc/using-data-api.rst | 664 ++++++++--------------------------------- 2 files changed, 128 insertions(+), 557 deletions(-) diff --git a/doc/datastore.rst b/doc/datastore.rst index 04a940d8cd5..77fd981d2df 100644 --- a/doc/datastore.rst +++ b/doc/datastore.rst @@ -33,7 +33,8 @@ queries over the spreadsheet contents. The DataStore Data API ====================== -The DataStore's Data API, which derives from the underlying data-table, is RESTful and JSON-based with extensive query capabilities. +The DataStore's Data API, which derives from the underlying data-table, +is RESTful and JSON-based with extensive query capabilities. Each resource in a CKAN instance has an associated DataStore 'table'. This table will be accessible via a web interface at:: @@ -45,28 +46,26 @@ For a detailed tutorial on using this API see :doc:`using-data-api`. Installation and Configuration ============================== -The DataStore in previous lives required a custom setup of ElasticSearch and Nginx, but that is no more, as it can use any relational database management system (PostgreSQL for example). +The DataStore in previous lives required a custom setup of ElasticSearch and Nginx, +but that is no more, as it can use any relational database management system +(PostgreSQL for example). To enable datastore features in CKAN ------------------------------------ -In your config file... - -Enable the ckan datastore:: - - ckan.datastore.enabled = 1 - -Ensure that the datastore extension is enabled:: +In your config file ensure that the datastore extension is enabled:: ckan.plugins = datastore -Ensure that the ckan.datastore_write_url variable is set:: +Also ensure that the ckan.datastore_write_url variable is set:: ckan.datastore_write_url = postgresql://ckanuser:pass@localhost/ckantest To test you can create a new datastore, so on linux command line do:: - curl -X POST http://127.0.0.1:5000/api/3/action/datastore_create -H "Authorization: {YOUR-API-KEY}" -d "{\"resource_id\": \"{PRE-EXISTING-RESOURCE-ID}\", \"fields\": [ {\"id\": \"a\"}, {\"id\": \"b\"} ], \"records\": [ { \"a\": 1, \"b\": \"xyz\"}, {\"a\": 2, \"b\": \"zzz\"} ]}" + curl -X POST http://127.0.0.1:5000/api/3/action/datastore_create -H "Authorization: {YOUR-API-KEY}" -d " + {\"resource_id\": \"{RESOURCE-ID}\", \"fields\": [ {\"id\": \"a\"}, {\"id\": \"b\"} ], + \"records\": [ { \"a\": 1, \"b\": \"xyz\"}, {\"a\": 2, \"b\": \"zzz\"} ]}" diff --git a/doc/using-data-api.rst b/doc/using-data-api.rst index f0a96172d08..ce7c72acebe 100644 --- a/doc/using-data-api.rst +++ b/doc/using-data-api.rst @@ -8,599 +8,171 @@ The following provides an introduction to using the CKAN :doc:`DataStore Introduction ============ -Each 'table' in the DataStore is an ElasticSearch_ index type ('table'). As -such the Data API for each CKAN resource is directly equivalent to a single -index 'type' in ElasticSearch (we tend to refer to it as a 'table'). - -This means you can (usually) directly re-use `ElasticSearch client libraries`_ -when connecting to a Data API endpoint. It also means that what follows is, in -essence, a tutorial in using the ElasticSearch_ API. - -The following short set of slides provide a brief overview and introduction to -the DataStore and the Data API. +The DataStore API allows tabular data to be stored inside CKAN quickly and easily. It is accessible through an interface accessible over HTTP and can be interacted with using JSON (the JavaScript Object Notation). .. raw:: html -.. _ElasticSearch: http://elasticsearch.org/ -.. _ElasticSearch client libraries: http://www.elasticsearch.org/guide/appendix/clients.html Quickstart ========== -``{{endpoint}}`` refers to the data API endpoint (or ElasticSearch index / -table). For example, on the DataHub_ this gold prices data resource -http://datahub.io/dataset/gold-prices/resource/b9aae52b-b082-4159-b46f-7bb9c158d013 -would have its Data API endpoint at: -http://datahub.io/api/data/b9aae52b-b082-4159-b46f-7bb9c158d013. If you were -just using ElasticSearch standalone an example of an endpoint would be: -http://localhost:9200/gold-prices/monthly-price-table. - -.. note:: every resource on a CKAN instance for which a DataStore table is - enabled provides links to its Data API endpoint via the Data API - button at the top right of the resource page. - -Key urls: +There are several endpoints into the DataStore API, they are: -* Query: ``{{endpoint}}/_search`` (in ElasticSearch < 0.19 this will return an - error if visited without a query parameter) +* datastore_create: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_create`` +* datastore_search: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search`` +* datastore_delete: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_delete`` - * Query example: ``{{endpoint}}/_search?size=5&pretty=true`` +API Reference +------------- -* Schema (Mapping): ``{{endpoint}}/_mapping`` +datastore_create +~~~~~~~~~~~~~~~~ -.. _DataHub: http://datahub.io/ - -Examples --------- - -cURL (or Browser) -~~~~~~~~~~~~~~~~~ - -The following examples utilize the cURL_ command line utility. If you prefer, -you you can just open the relevant urls in your browser:: - - // query for documents / rows with title field containing 'jones' - // added pretty=true to get the json results pretty printed - curl {{endpoint}}/_search?q=title:jones&size=5&pretty=true - -Adding some data (requires an :ref:`API Key `):: - - // requires an API key - // Data (argument to -d) should be a JSON document - curl -X POST -H "Authorization: {{YOUR-API-KEY}}" {{endpoint}} -d '{ - "title": "jones", - "amount": 5.7 - }' - -.. _cURL: http://curl.haxx.se/ - -Javascript -~~~~~~~~~~ - -A simple ajax (JSONP) request to the data API using jQuery:: - - var data = { - size: 5 // get 5 results - q: 'title:jones' // query on the title field for 'jones' - }; - $.ajax({ - url: {{endpoint}}/_search, - dataType: 'jsonp', - success: function(data) { - alert('Total results found: ' + data.hits.total) - } - }); - -The Data API supports CORs so you can also write to it (this requires the json2_ library for ``JSON.stringify``):: - - var data = { - title: 'jones', - amount: 5.7 - }; - $.ajax({ - url: {{endpoint}}, - type: 'POST', - data: JSON.stringify(data), - success: function(data) { - alert('Uploaded ok') - } - }); - -.. _json2: https://github.com/douglascrockford/JSON-js/blob/master/json2.js - -Python -~~~~~~ +The datastore_create API endpoint allows a user to post JSON data to +be stored against a resource, the JSON must be in the following form:: -.. note:: You can also use the `DataStore Python client library`_. + { + resource_id: resource_id, # the data is going to be stored against. + fields: [], # a list of dictionaries of fields/columns and their extra metadata. + records: [], # a list of dictionaries of the data eg [{"dob": "2005", "some_stuff": ['a', b']}, ..] + } -.. _DataStore Python client library: http://github.com/okfn/datastore-client -:: +datastore_search +~~~~~~~~~~~~~~~~ - import urllib2 - import json - - # ================================= - # Store data in the DataStore table - - url = '{{endpoint}}' - data = { - 'title': 'jones', - 'amount': 5.7 - } - # have to send the data as JSON - data = json.dumps(data) - # need to add your API key (and have authorization to write to this endpoint) - headers = {'Authorization': 'YOUR-API-KEY'} - - req = urllib2.Request(url, data, headers) - out = urllib2.urlopen(req) - print out.read() - - # ========================= - # Query the DataStore table - - url = '{{endpoint}}/_search?q=title:jones&size=5' - req = urllib2.Request(url) - out = urllib2.urlopen(req) - data = out.read() - print data - # returned data is JSON - data = json.loads(data) - # total number of results - print data['hits']['total'] - -Querying -======== - -Basic Queries Using Only the Query String ------------------------------------------ - -Basic queries can be done using only query string parameters in the URL. For -example, the following searches for text 'hello' in any field in any document -and returns at most 5 results:: - - {{endpoint}}/_search?q=hello&size=5 - -Basic queries like this have the advantage that they only involve accessing a -URL and thus, for example, can be performed just using any web browser. -However, this method is limited and does not give you access to most of the -more powerful query features. - -Basic queries use the `q` query string parameter which supports the `Lucene -query parser syntax`_ and hence filters on specific fields (e.g. `fieldname:value`), wildcards (e.g. `abc*`) and more. - -.. _Lucene query parser syntax: http://lucene.apache.org/core/old_versioned_docs/versions/3_0_0/queryparsersyntax.html - -There are a variety of other options (e.g. size, from etc) that you can also -specify to customize the query and its results. Full details can be found in -the `ElasticSearch URI request docs`_. - -.. _ElasticSearch URI request docs: http://www.elasticsearch.org/guide/reference/api/search/uri-request.html - -Full Query API --------------- - -More powerful and complex queries, including those that involve faceting and -statistical operations, should use the full ElasticSearch query language and API. - -In the query language queries are written as a JSON structure and is then sent -to the query endpoint (details of the query langague below). There are two -options for how a query is sent to the search endpoint: - -1. Either as the value of a source query parameter e.g.:: - - {{endpoint}}/_search?source={Query-as-JSON} - -2. Or in the request body, e.g.:: - - curl -XGET {{endpoint}}/_search -d 'Query-as-JSON' - - For example:: - - curl -XGET {{endpoint}}/_search -d '{ - "query" : { - "term" : { "user": "kimchy" } - } - }' +The datastore_search API endpoint allows a user to search data at a resource, +the JSON for searching must be in the following form:: + { + resource_id: # the resource id to be searched against + filters : # dictionary of matching conditions to select e.g {'key1': 'a. 'key2': 'b'} + # this will be equivalent to "select * from table where key1 = 'a' and key2 = 'b' " + q: # full text query + limit: # limit the amount of rows to size defaults to 20 + offset: # offset the amount of rows + fields: # list of fields return in that order, defaults (empty or not present) to all fields in fields order. + sort: + } -Query Language -============== +datastore_delete +~~~~~~~~~~~~~~~~ -Queries are JSON objects with the following structure (each of the main -sections has more detail below):: +The datastore_delete API endpoint allows a user to delete from a resource, +the JSON for searching must be in the following form:: - { - size: # number of results to return (defaults to 10) - from: # offset into results (defaults to 0) - fields: # list of document fields that should be returned - http://elasticsearch.org/guide/reference/api/search/fields.html - sort: # define sort order - see http://elasticsearch.org/guide/reference/api/search/sort.html + { + resource_id: resource_id # the data that is going to be deleted. + filters: # dictionary of matching conditions to delete + # e.g {'key1': 'a. 'key2': 'b'} + # this will be equivalent to "delete from table where key1 = 'a' and key2 = 'b' " + } - query: { - # "query" object following the Query DSL: http://elasticsearch.org/guide/reference/query-dsl/ - # details below - }, - - facets: { - # facets specifications - # Facets provide summary information about a particular field or fields in the data - } - - # special case for situations where you want to apply filter/query to results but *not* to facets - filter: { - # filter objects - # a filter is a simple "filter" (query) on a specific field. - # Simple means e.g. checking against a specific value or range of values - }, - } - -Query results look like:: - - { - # some info about the query (which shards it used, how long it took etc) - ... - # the results - hits: { - total: # total number of matching documents - hits: [ - # list of "hits" returned - { - _id: # id of document - score: # the search index score - _source: { - # document 'source' (i.e. the original JSON document you sent to the index - } - } - ] - } - # facets if these were requested - facets: { - ... - } - } - -Query DSL: Overview -------------------- - -Query objects are built up of sub-components. These sub-components are either -basic or compound. Compound sub-components may contains other sub-components -while basic may not. Example:: - - { - "query": { - # compound component - "bool": { - # compound component - "must": { - # basic component - "term": { - "user": "jones" - } - } - # compound component - "must_not": { - # basic component - "range" : { - "age" : { - "from" : 10, - "to" : 20 - } - } - } - } - } - } - -In addition, and somewhat confusingly, ElasticSearch distinguishes between -sub-components that are "queries" and those that are "filters". Filters, are -really special kind of queries that are: mostly basic (though boolean -compounding is alllowed); limited to one field or operation and which, as such, -are especially performant. - -Examples, of filters are (full list on RHS at the bottom of the query-dsl_ page): - - * term: filter on a value for a field - * range: filter for a field having a range of values (>=, <= etc) - * geo_bbox: geo bounding box - * geo_distance: geo distance - -.. _query-dsl: http://elasticsearch.org/guide/reference/query-dsl/ - -Rather than attempting to set out all the constraints and options of the -query-dsl we now offer a variety of examples. Examples -------- -Match all / Find Everything -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:: +Some of the following commands require obtaining an :ref:`API Key `. - { - "query": { - "match_all": {} - } - } - -Classic Search-Box Style Full-Text Query -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This will perform a full-text style query across all fields. The query string -supports the `Lucene query parser syntax`_ and hence filters on specific fields -(e.g. `fieldname:value`), wildcards (e.g. `abc*`) as well as a variety of -options. For full details see the query-string_ documentation. - -:: - - { - "query": { - "query_string": { - "query": {query string} - } - } - } - -.. _query-string: http://elasticsearch.org/guide/reference/query-dsl/query-string-query.html - -Filter on One Field -~~~~~~~~~~~~~~~~~~~ - -:: +cURL (or Browser) +~~~~~~~~~~~~~~~~~ - { - "query": { - "term": { - {field-name}: {value} - } - } - } - -High performance equivalent using filters:: - - { - "query": { - "constant_score": { - "filter": { - "term": { - # note that value should be *lower-cased* - {field-name}: {value} - } - } - } - } - -Find all documents with value in a range -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This can be used both for text ranges (e.g. A to Z), numeric ranges (10-20) and -for dates (ElasticSearch will converts dates to ISO 8601 format so you can -search as 1900-01-01 to 1920-02-03). - -:: - - { - "query": { - "constant_score": { - "filter": { - "range": { - {field-name}: { - "from": {lower-value} - "to": {upper-value} - } - } - } - } - } - } +The following examples utilize the cURL_ command line utility. If you prefer, +you you can just open the relevant urls in your browser:: -For more details see `range filters`_. + # This creates a datastore + curl -X POST {ENDPOINT:datastore_create} -H "Authorization: {YOUR-API-KEY}" -d " + {\"resource_id\": \"{RESOURCE-ID}\", \"fields\": [ {\"id\": \"a\"}, {\"id\": \"b\"} ], + \"records\": [ { \"a\": 1, \"b\": \"xyz\"}, {\"a\": 2, \"b\": \"zzz\"} ]}" -.. _range filters: http://elasticsearch.org/guide/reference/query-dsl/range-filter.html + #This queries a datastore + curl -X POST {ENDPOINT:datastore_search} -H "Authorization: {YOUR-API-KEY}" -d " + {\"resource_id\": \"{RESOURCE-ID}\" }" -Full-Text Query plus Filter on a Field -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _cURL: http://curl.haxx.se/ -:: +Javascript +~~~~~~~~~~ - { - "query": { - "query_string": { - "query": {query string} - }, - "term": { - {field}: {value} - } +.. + A simple ajax (JSONP) request to the data API using jQuery:: + + var data = { + size: 5 // get 5 results + q: 'title:jones' // query on the title field for 'jones' + }; + $.ajax({ + url: {{endpoint}}/_search, + dataType: 'jsonp', + success: function(data) { + alert('Total results found: ' + data.hits.total) } - } - - -Filter on two fields -~~~~~~~~~~~~~~~~~~~~ - -Note that you cannot, unfortunately, have a simple and query by adding two -filters inside the query element. Instead you need an 'and' clause in a filter -(which in turn requires nesting in 'filtered'). You could also achieve the same -result here using a `bool query`_. - -.. _bool query: http://elasticsearch.org/guide/reference/query-dsl/bool-query.html - -:: - - { - "query": { - "filtered": { - "query": { - "match_all": {} - }, - "filter": { - "and": [ - { - "range" : { - "b" : { - "from" : 4, - "to" : "8" - } - }, - }, - { - "term": { - "a": "john" - } - } - ] - } - } + }); + + The Data API supports CORs so you can also write to it (this requires the json2_ library for ``JSON.stringify``):: + + var data = { + title: 'jones', + amount: 5.7 + }; + $.ajax({ + url: {{endpoint}}, + type: 'POST', + data: JSON.stringify(data), + success: function(data) { + alert('Uploaded ok') } - } - -Geospatial Query to find results near a given point -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This uses the `Geo Distance filter`_. It requires that indexed documents have a field of `geo point type`_. - -.. _Geo Distance filter: http://www.elasticsearch.org/guide/reference/query-dsl/geo-distance-filter.html -.. _geo point type: http://www.elasticsearch.org/guide/reference/mapping/geo-point-type.html - -Source data (a point in San Francisco!):: - - # This should be in lat,lon order - { - ... - "Location": "37.7809035011582, -122.412119695795" - } - -There are alternative formats to provide lon/lat locations e.g. (see ElasticSearch documentation for more):: - - # Note this must have lon,lat order (opposite of previous example!) - { - "Location":[-122.414753390488, 37.7762147914147] - } - - # or ... - { - "Location": { - "lon": -122.414753390488, - "lat": 37.7762147914147 - } - } - -We also need a mapping to specify that Location field is of type geo_point as this will not usually get guessed from the data (see below for more on mappings):: - - "properties": { - "Location": { - "type": "geo_point" - } - ... - } - -Now the actual query:: - - { - "query": { - "filtered" : { - "query" : { - "match_all" : {} - }, - "filter" : { - "geo_distance" : { - "distance" : "20km", - "Location" : { - "lat" : 37.776, - "lon" : -122.41 - } - } - } - } - } - } - -Note that you can specify the query using specific lat, lon attributes even -though original data did not have this structure (you can also use a query -similar to the original structure if you wish - see `Geo distance filter`_ for -more information). - - -Facets ------- - -Facets provide a way to get summary information about then data in an -elasticsearch table, for example counts of distinct values. - -ElasticSearch (and hence the Data API) provides rich faceting capabilities: -http://www.elasticsearch.org/guide/reference/api/search/facets/ + }); -There are various kinds of facets available, for example (full list on the facets page): + .. _json2: https://github.com/douglascrockford/JSON-js/blob/master/json2.js -* Terms_ - counts by distinct terms (values) in a field -* Range_ - counts for a given set of ranges in a field -* Histogram_ and `Date Histogram`_ - counts by constant interval ranges -* Statistical_ - statistical summary of a field (mean, sum etc) -* `Terms Stats`_ - statistical summary on one field (stats field) for distinct - terms in another field. For example, spending stats per department or per - region. -* `Geo Distance`_: counts by distance ranges from a given point - -Note that you can apply multiple facets per query. - -.. _Terms: http://www.elasticsearch.org/guide/reference/api/search/facets/terms-facet.html -.. _Range: http://www.elasticsearch.org/guide/reference/api/search/facets/range-facet.html -.. _Histogram: http://www.elasticsearch.org/guide/reference/api/search/facets/histogram-facet.html -.. _Date Histogram: http://www.elasticsearch.org/guide/reference/api/search/facets/date-histogram-facet.html -.. _Statistical: http://www.elasticsearch.org/guide/reference/api/search/facets/statistical-facet.html -.. _Terms Stats: http://www.elasticsearch.org/guide/reference/api/search/facets/terms-stats-facet.html -.. _Geo Distance: http://www.elasticsearch.org/guide/reference/api/search/facets/geo-distance-facet.html - - -Adding, Updating and Deleting Data -================================== - -ElasticSeach, and hence the Data API, have a standard RESTful API. Thus:: - - POST {{endpoint}} : INSERT - PUT/POST {{endpoint}}/{{id}} : UPDATE (or INSERT) - DELETE {{endpoint}}/{{id}} : DELETE - -For more on INSERT and UPDATE see the `Index API`_ documentation. - -.. _Index API: http://elasticsearch.org/guide/reference/api/index_.html - -There is also support bulk insert and updates via the `Bulk API`_. +Python +~~~~~~ -.. _Bulk API: http://elasticsearch.org/guide/reference/api/bulk.html +.. + note:: You can also use the `DataStore Python client library`_. -.. note:: The `DataStore Python client library`_ has support for inserting, - updating (in bulk) and deleting. There is also support for these - operations in the ReclineJS javascript library. + _DataStore Python client library: http://github.com/okfn/datastore-client + :: -Schema Mapping -============== + import urllib2 + import json -As the ElasticSearch documentation states: + # ================================= + # Store data in the DataStore table - Mapping is the process of defining how a document should be mapped to the - Search Engine, including its searchable characteristics such as which fields - are searchable and if/how they are tokenized. In ElasticSearch, an index may - store documents of different “mapping types”. ElasticSearch allows one to - associate multiple mapping definitions for each mapping type. + url = '{{endpoint}}' + data = { + 'title': 'jones', + 'amount': 5.7 + } + # have to send the data as JSON + data = json.dumps(data) + # need to add your API key (and have authorization to write to this endpoint) + headers = {'Authorization': 'YOUR-API-KEY'} - Explicit mapping is defined on an index/type level. By default, there isn't a - need to define an explicit mapping, since one is automatically created and - registered when a new type or new field is introduced (with no performance - overhead) and have sensible defaults. Only when the defaults need to be - overridden must a mapping definition be provided. + req = urllib2.Request(url, data, headers) + out = urllib2.urlopen(req) + print out.read() -Relevant docs: http://elasticsearch.org/guide/reference/mapping/. + # ========================= + # Query the DataStore table + url = '{{endpoint}}/_search?q=title:jones&size=5' + req = urllib2.Request(url) + out = urllib2.urlopen(req) + data = out.read() + print data + # returned data is JSON + data = json.loads(data) + # total number of results + print data['hits']['total'] -JSONP support -============= -JSONP support is available on any request via a simple callback query string parameter:: +PHP +~~~~~~ - ?callback=my_callback_name From ab7ded798bde1b09ed3769c6b8f91ee44fd411a9 Mon Sep 17 00:00:00 2001 From: Daniel Lewis Date: Fri, 10 Aug 2012 14:45:18 +0100 Subject: [PATCH 050/130] Added python code to using-data-api.rst and made some other minor adjustments on the page, including some detail about creating a dataset+resource --- doc/using-data-api.rst | 105 +++++++++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 40 deletions(-) diff --git a/doc/using-data-api.rst b/doc/using-data-api.rst index ce7c72acebe..5f676ed81c8 100644 --- a/doc/using-data-api.rst +++ b/doc/using-data-api.rst @@ -24,6 +24,33 @@ There are several endpoints into the DataStore API, they are: * datastore_search: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search`` * datastore_delete: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_delete`` +Before going into detail about the API and the examples, it is useful to create a resource first so that you can store data against it in the Datastore. This can be done either through the CKAN Graphical User Interface. + +Using the data API, we need to send JSON with this structure:: + + { + id: Uuid, + name: Name-String, + title: String, + version: String, + url: String, + resources: [ { url: String, format: String, description: String, hash: String }, ...], + author: String, + author_email: String, + maintainer: String, + maintainer_email: String, + license_id: String, + tags: Tag-List, + notes: String, + extras: { Name-String: String, ... } + } + +To the following endpoint: + +* Dataset Model Endpoint: ``http://{YOUR-CKAN-INSTALLATION}/api/rest/dataset`` + +More details about creating a resource through the Data API are available on the :ref:`CKAN API page ` + API Reference ------------- @@ -96,6 +123,8 @@ you you can just open the relevant urls in your browser:: Javascript ~~~~~~~~~~ +Coming soon... + .. A simple ajax (JSONP) request to the data API using jQuery:: @@ -131,48 +160,44 @@ Javascript Python ~~~~~~ -.. - note:: You can also use the `DataStore Python client library`_. - - _DataStore Python client library: http://github.com/okfn/datastore-client - - :: - - import urllib2 - import json - - # ================================= - # Store data in the DataStore table - - url = '{{endpoint}}' - data = { - 'title': 'jones', - 'amount': 5.7 - } - # have to send the data as JSON - data = json.dumps(data) - # need to add your API key (and have authorization to write to this endpoint) - headers = {'Authorization': 'YOUR-API-KEY'} - - req = urllib2.Request(url, data, headers) - out = urllib2.urlopen(req) - print out.read() - - # ========================= - # Query the DataStore table - - url = '{{endpoint}}/_search?q=title:jones&size=5' - req = urllib2.Request(url) - out = urllib2.urlopen(req) - data = out.read() - print data - # returned data is JSON - data = json.loads(data) - # total number of results - print data['hits']['total'] - +Using the Python Requests_ library we can create a datastore like this:: + + #! /usr/bin/env python + + import requests + import json + + auth_key = '' + + url = "http://127.0.0.1:5000/api/3/action/" # An example "action" endpoint + + datastore_structure = { + 'resource_id': '', + 'fields': [ {"id": "a"}, {"id": "b"} ], + "records": [ { "a": 1, "b": "xyz"}, {"a": 2, "b": "zzz"} ] + } + headers = {'content-type': 'application/json', 'Authorization': auth_key} + r = requests.post(url + 'datastore_create', data=json.dumps(datastore_structure), headers=headers) + print "done, and now for a quick search\n" + + datastore_structure = { + 'resource_id': '' + } + headers = {'content-type': 'application/json', 'Authorization': auth_key} + r = requests.post(url + 'datastore_search', data=json.dumps(datastore_structure), headers=headers) + + print r.text + + print "done\n" + + +Python urllib2 version Coming soon... + + +.. _Requests: http://docs.python-requests.org/ PHP ~~~~~~ +Coming soon... From 31a751969c5209a5db7c9dd4d4c96497c5efb16d Mon Sep 17 00:00:00 2001 From: Daniel Lewis Date: Mon, 13 Aug 2012 11:26:49 +0100 Subject: [PATCH 051/130] Changed around some things in datastore and using the data api pages, as requested by Rufus --- doc/datastore.rst | 48 ++++++++++++++++++++++++++++++++++++++++++ doc/using-data-api.rst | 48 +----------------------------------------- 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/doc/datastore.rst b/doc/datastore.rst index 77fd981d2df..26a81103b53 100644 --- a/doc/datastore.rst +++ b/doc/datastore.rst @@ -94,3 +94,51 @@ How It Works (Technically) 3. (Assuming OK) CKAN hands (internally) to the database querying system which handles the request +.. _datastoreapiref: + +API Reference +------------- + +datastore_create +~~~~~~~~~~~~~~~~ + +The datastore_create API endpoint allows a user to post JSON data to +be stored against a resource, the JSON must be in the following form:: + + { + resource_id: resource_id, # the data is going to be stored against. + fields: [], # a list of dictionaries of fields/columns and their extra metadata. + records: [], # a list of dictionaries of the data eg [{"dob": "2005", "some_stuff": ['a', b']}, ..] + } + + +datastore_search +~~~~~~~~~~~~~~~~ + +The datastore_search API endpoint allows a user to search data at a resource, +the JSON for searching must be in the following form:: + + { + resource_id: # the resource id to be searched against + filters : # dictionary of matching conditions to select e.g {'key1': 'a. 'key2': 'b'} + # this will be equivalent to "select * from table where key1 = 'a' and key2 = 'b' " + q: # full text query + limit: # limit the amount of rows to size defaults to 20 + offset: # offset the amount of rows + fields: # list of fields return in that order, defaults (empty or not present) to all fields in fields order. + sort: + } + +datastore_delete +~~~~~~~~~~~~~~~~ + +The datastore_delete API endpoint allows a user to delete from a resource, +the JSON for searching must be in the following form:: + + { + resource_id: resource_id # the data that is going to be deleted. + filters: # dictionary of matching conditions to delete + # e.g {'key1': 'a. 'key2': 'b'} + # this will be equivalent to "delete from table where key1 = 'a' and key2 = 'b' " + } + diff --git a/doc/using-data-api.rst b/doc/using-data-api.rst index 5f676ed81c8..73dcb1df959 100644 --- a/doc/using-data-api.rst +++ b/doc/using-data-api.rst @@ -49,53 +49,7 @@ To the following endpoint: * Dataset Model Endpoint: ``http://{YOUR-CKAN-INSTALLATION}/api/rest/dataset`` -More details about creating a resource through the Data API are available on the :ref:`CKAN API page ` - -API Reference -------------- - -datastore_create -~~~~~~~~~~~~~~~~ - -The datastore_create API endpoint allows a user to post JSON data to -be stored against a resource, the JSON must be in the following form:: - - { - resource_id: resource_id, # the data is going to be stored against. - fields: [], # a list of dictionaries of fields/columns and their extra metadata. - records: [], # a list of dictionaries of the data eg [{"dob": "2005", "some_stuff": ['a', b']}, ..] - } - - -datastore_search -~~~~~~~~~~~~~~~~ - -The datastore_search API endpoint allows a user to search data at a resource, -the JSON for searching must be in the following form:: - - { - resource_id: # the resource id to be searched against - filters : # dictionary of matching conditions to select e.g {'key1': 'a. 'key2': 'b'} - # this will be equivalent to "select * from table where key1 = 'a' and key2 = 'b' " - q: # full text query - limit: # limit the amount of rows to size defaults to 20 - offset: # offset the amount of rows - fields: # list of fields return in that order, defaults (empty or not present) to all fields in fields order. - sort: - } - -datastore_delete -~~~~~~~~~~~~~~~~ - -The datastore_delete API endpoint allows a user to delete from a resource, -the JSON for searching must be in the following form:: - - { - resource_id: resource_id # the data that is going to be deleted. - filters: # dictionary of matching conditions to delete - # e.g {'key1': 'a. 'key2': 'b'} - # this will be equivalent to "delete from table where key1 = 'a' and key2 = 'b' " - } +More details about creating a resource through the Data API are available on the :ref:`CKAN API page `. More information about the Datastore API can be found on the :ref:`datastore page `. Examples From 0525c9862129a85e55b859e6236614b55075d891 Mon Sep 17 00:00:00 2001 From: Daniel Lewis Date: Mon, 13 Aug 2012 12:35:26 +0100 Subject: [PATCH 052/130] Using Data API documentation now reflects GET-based datastore_search, and also includes a Python URLLib2 example --- doc/using-data-api.rst | 47 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/doc/using-data-api.rst b/doc/using-data-api.rst index 73dcb1df959..6cf023183f6 100644 --- a/doc/using-data-api.rst +++ b/doc/using-data-api.rst @@ -69,8 +69,7 @@ you you can just open the relevant urls in your browser:: \"records\": [ { \"a\": 1, \"b\": \"xyz\"}, {\"a\": 2, \"b\": \"zzz\"} ]}" #This queries a datastore - curl -X POST {ENDPOINT:datastore_search} -H "Authorization: {YOUR-API-KEY}" -d " - {\"resource_id\": \"{RESOURCE-ID}\" }" + curl {ENDPOINT:datastore_search}?resource_id={RESOURCE-ID} -H "Authorization: {YOUR-API-KEY}" .. _cURL: http://curl.haxx.se/ @@ -114,6 +113,47 @@ Coming soon... Python ~~~~~~ +A Python URLLib2 datastore_create and datastore_search would look like:: + + #! /usr/bin/env python + import urllib + import urllib2 + import json + + auth_key = '{YOUR-AUTH-KEY}' + + # In python using urllib2 for datastore_create it is... + + url = "http://127.0.0.1:5000/api/3/action/" + + datastore_structure = { + 'resource_id': '{RESOURCE-ID}', + 'fields': [ {"id": "a"}, {"id": "b"} ], + "records": [ { "a": 12, "b": "abc"}, {"a": 2, "b": "zzz"} ] + } + headers = {'content-type': 'application/json', 'Authorization': auth_key} + + + + req = urllib2.Request(url + 'datastore_create', data=json.dumps(datastore_structure), headers=headers) + response = urllib2.urlopen(req) + + + # in python for datastore_search using urllib2.... + + datastore_structure = { + 'resource_id': '{RESOURCE-ID}' + } + + url_values = urllib.urlencode(datastore_structure) + req = urllib2.Request(url + 'datastore_search?' + url_values, headers=headers) + response = urllib2.urlopen(req) + + print response.read() + + print "done\n" + + Using the Python Requests_ library we can create a datastore like this:: #! /usr/bin/env python @@ -145,9 +185,6 @@ Using the Python Requests_ library we can create a datastore like this:: print "done\n" -Python urllib2 version Coming soon... - - .. _Requests: http://docs.python-requests.org/ PHP From f9f8b3049ff956eaead72b161644745493a76c5f Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 14 Aug 2012 11:18:08 +0100 Subject: [PATCH 053/130] [#2618] Rename config.ini resource.config to avoid .gitignore rules --- ckan/html_resources/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index 65e1bbfe4ba..6a0b0f27515 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -1,5 +1,5 @@ ''' This file creates fanstatic resources from the sub directories. The -directory can contain a config.ini to specify how the resources should +directory can contain a resource.config to specify how the resources should be treated. minified copies of the resources are created if the resource has a later modification time than existing minified versions. @@ -7,7 +7,7 @@ ckan setup.py file. -config.ini (example) +resource.config (example) ========== # all resources are named without their file extension [main] @@ -230,7 +230,7 @@ def key(resource): def create_library(name, path): ''' Creates a fanstatic library `name` with the contents of a - directory `path` using config.ini if found. Files are minified + directory `path` using resource.config if found. Files are minified if needed. ''' def min_path(path): @@ -277,7 +277,7 @@ def create_resource(path, lib_name, count): if path in dont_bundle: kw['dont_bundle'] = True kw['custom_order'] = count - # FIXME needs config.ini options enabled + # FIXME needs resource.config options enabled if False: other_browsers = False condition = '' @@ -296,10 +296,10 @@ def create_resource(path, lib_name, count): depends = {} groups = {} - # parse the config.ini file if it exists + # parse the resource.config file if it exists resource_path = os.path.dirname(__file__) resource_path = os.path.join(resource_path, path) - config_path = os.path.join(resource_path, 'config.ini') + config_path = os.path.join(resource_path, 'resource.config') if os.path.exists(config_path): config = ConfigParser.RawConfigParser() config.read(config_path) From 07305f6894f696b5b830cdbf3fc3c7b9b86cece2 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 14 Aug 2012 11:18:37 +0100 Subject: [PATCH 054/130] [#2618] Add resource.config for javascript dir --- ckan/public/base/javascript/resource.config | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 ckan/public/base/javascript/resource.config diff --git a/ckan/public/base/javascript/resource.config b/ckan/public/base/javascript/resource.config new file mode 100644 index 00000000000..604e623432f --- /dev/null +++ b/ckan/public/base/javascript/resource.config @@ -0,0 +1,25 @@ + +[groups] + +main = + plugins/jquery.inherit.js + plugins/jquery.proxy-all.js + plugins/jquery.url-helpers.js + plugins/jquery.date-helpers.js + plugins/jquery.slug.js + plugins/jquery.slug-preview.js + plugins/jquery.form-warning.js + sandbox.js + module.js + pubsub.js + client.js + notify.js + i18n.js + main.js + modules/select-switch.js + modules/slug-preview.js + modules/basic-form.js + modules/confirm-delete.js + modules/api-info.js + modules/autocomplete.js + modules/custom-fields.js From 0c700248d634e15f1a15cbb6380fcee404221601 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 14 Aug 2012 11:19:06 +0100 Subject: [PATCH 055/130] [#2618] Use order of groups if supplied --- ckan/html_resources/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index 6a0b0f27515..66864a33233 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -332,6 +332,12 @@ def create_resource(path, lib_name, count): minify(f, dirname, cssmin) file_list.append(filepath) + # if groups are defined make sure the order supplied there is honored + for group in groups: + for resource in groups[group]: + if resource not in order: + order.append(resource) + for x in reversed(order): if x in file_list: file_list.remove(x) From 4e8195a4dcfa28fef8018b832155bff77a1bdf0e Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 14 Aug 2012 12:03:12 +0100 Subject: [PATCH 056/130] [#2618] Improve ordering of libs --- ckan/html_resources/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index 66864a33233..001b7187e9f 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -218,9 +218,9 @@ def sort_resources(resources): def key(resource): return ( resource.order, - resource.custom_order, resource.library.library_nr, resource.library.name, + resource.custom_order, resource.dependency_nr, resource.relpath) return sorted(resources, key=key) From 987dde50d52a330641f737ba89f6e9e736d1c012 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 14 Aug 2012 12:04:41 +0100 Subject: [PATCH 057/130] [#2618] Look at html shim and other vender stuff --- ckan/config/middleware.py | 2 +- ckan/html_resources/__init__.py | 3 +++ ckan/templates/base.html | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index ecd2065406e..a3fb74e9839 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -69,7 +69,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf): #app = QueueLogMiddleware(app) # Fanstatic - if asbool(config.get('debug', False)): + if not asbool(config.get('debug', False)): fanstatic_config = { 'versioning' : True, 'recompute_hashes' : True, diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index 001b7187e9f..e11f9ee9da0 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -288,6 +288,7 @@ def create_resource(path, lib_name, count): resource = Resource(library, filename, **kw) # add the resource to this module fanstatic_name = '%s/%s' % (lib_name, filename) + print 'create resource %s' % fanstatic_name setattr(module, fanstatic_name, resource) return resource @@ -365,6 +366,8 @@ def create_resource(path, lib_name, count): base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'public', 'base', 'javascript')) create_library('base', base_path) +base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'public', 'base', 'vendor')) +create_library('vendor', base_path) ### create our libraries here from any subdirectories ##for dirname, dirnames, filenames in os.walk(os.path.dirname(__file__)): diff --git a/ckan/templates/base.html b/ckan/templates/base.html index 6c2b8c3cc25..eae6cc1c43f 100644 --- a/ckan/templates/base.html +++ b/ckan/templates/base.html @@ -55,7 +55,8 @@ #} {%- block html5shim -%} {# Temporary link until we get resource helper working #} - + {# #} + {% resource 'vendor/html5.js' %} {%- endblock -%} {# From b8eb66d861121c3cb7775370d7ff742c600321c7 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Tue, 14 Aug 2012 14:55:53 +0100 Subject: [PATCH 058/130] [doc/datastore][s]: minor refactoring and tidying of the docs. --- doc/datastore.rst | 43 +++++++++--------------------------------- doc/using-data-api.rst | 6 +----- 2 files changed, 10 insertions(+), 39 deletions(-) diff --git a/doc/datastore.rst b/doc/datastore.rst index 26a81103b53..aac6cb2c67a 100644 --- a/doc/datastore.rst +++ b/doc/datastore.rst @@ -6,16 +6,6 @@ The CKAN DataStore provides a database for structured storage of data together with a powerful Web-accesible Data API, all seamlessly integrated into the CKAN interface and authorization system. -Overview -======== - -The following short set of slides provide a brief overview and introduction to -the DataStore and the Data API. - -.. raw:: html - - - Relationship to FileStore ========================= @@ -36,12 +26,9 @@ The DataStore Data API The DataStore's Data API, which derives from the underlying data-table, is RESTful and JSON-based with extensive query capabilities. -Each resource in a CKAN instance has an associated DataStore 'table'. This -table will be accessible via a web interface at:: - - /api/data/{resource-id} - -For a detailed tutorial on using this API see :doc:`using-data-api`. +Each resource in a CKAN instance can have an associated DataStore 'table'. The +basic API for accessing the DataStore is detailed below. For a detailed +tutorial on using this API see :doc:`using-data-api`. Installation and Configuration ============================== @@ -63,18 +50,16 @@ Also ensure that the ckan.datastore_write_url variable is set:: To test you can create a new datastore, so on linux command line do:: - curl -X POST http://127.0.0.1:5000/api/3/action/datastore_create -H "Authorization: {YOUR-API-KEY}" -d " - {\"resource_id\": \"{RESOURCE-ID}\", \"fields\": [ {\"id\": \"a\"}, {\"id\": \"b\"} ], - \"records\": [ { \"a\": 1, \"b\": \"xyz\"}, {\"a\": 2, \"b\": \"zzz\"} ]}" + curl -X POST http://127.0.0.1:5000/api/3/action/datastore_create -H "Authorization: {YOUR-API-KEY}" + -d '{"resource_id": "{RESOURCE-ID}", "fields": [ {"id": "a"}, {"id": "b"} ], + "records": [ { "a": 1, "b": "xyz"}, {"a": 2, "b": "zzz"} ]}' - -.. _datastorer: - DataStorer: Automatically Add Data to the DataStore =================================================== -Often, one wants data that is added to CKAN (whether it is linked to or uploaded to the :doc:`FileStore `) to be automatically added to the +Often, one wants data that is added to CKAN (whether it is linked to or +uploaded to the :doc:`FileStore `) to be automatically added to the DataStore. This requires some processing, to extract the data from your files and to add it to the DataStore in the format the DataStore can handle. @@ -83,18 +68,8 @@ performed by a DataStorer, a queue process that runs asynchronously and can be triggered by uploads or other activities. The DataStorer is an extension and can be found, along with installation instructions, at: -https://github.com/okfn/ckanext-datastorer - - -How It Works (Technically) -========================== - -1. Request arrives at e.g. /dataset/{id}/resource/{resource-id}/data -2. CKAN checks authentication and authorization. -3. (Assuming OK) CKAN hands (internally) to the database querying system which handles the - request +.. _datastorer: https://github.com/okfn/ckanext-datastorer -.. _datastoreapiref: API Reference ------------- diff --git a/doc/using-data-api.rst b/doc/using-data-api.rst index 6cf023183f6..ac5b2cc97e6 100644 --- a/doc/using-data-api.rst +++ b/doc/using-data-api.rst @@ -10,10 +10,6 @@ Introduction The DataStore API allows tabular data to be stored inside CKAN quickly and easily. It is accessible through an interface accessible over HTTP and can be interacted with using JSON (the JavaScript Object Notation). -.. raw:: html - - - Quickstart ========== @@ -49,7 +45,7 @@ To the following endpoint: * Dataset Model Endpoint: ``http://{YOUR-CKAN-INSTALLATION}/api/rest/dataset`` -More details about creating a resource through the Data API are available on the :ref:`CKAN API page `. More information about the Datastore API can be found on the :ref:`datastore page `. +More details about creating a resource through the Data API are available on the :ref:`CKAN API page `. More information about the Datastore API can be found on the :doc:`datastore page `. Examples From 067431895f98496e6ad15a46357620bdbc7e9cde Mon Sep 17 00:00:00 2001 From: kindly Date: Wed, 15 Aug 2012 11:10:02 +0100 Subject: [PATCH 059/130] [#2733] add 60 second query time out --- ckanext/datastore/db.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index dd1d8fc4155..8c85267d46a 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -478,12 +478,15 @@ def create(context, data_dict): ''' engine = _get_engine(context, data_dict) context['connection'] = engine.connect() + timeout = context.get('query_timeout', 60000) _cache_types(context) # close connection at all cost. try: # check if table already existes trans = context['connection'].begin() + context['connection'].execute( + 'set local statement_timeout to {}'.format(timeout)) result = context['connection'].execute( 'select * from pg_tables where tablename = %s', data_dict['resource_id'] @@ -495,8 +498,11 @@ def create(context, data_dict): insert_data(context, data_dict) trans.commit() return data_dict - except: - trans.rollback() + except Exception, e: + if 'due to statement timeout' in str(e): + raise p.toolkit.ValidationError({ + 'query': 'Query took too long' + }) raise finally: context['connection'].close() @@ -528,7 +534,7 @@ def delete(context, data_dict): trans.commit() return data_dict - except: + except Exception, e: trans.rollback() raise finally: @@ -538,10 +544,13 @@ def delete(context, data_dict): def search(context, data_dict): engine = _get_engine(context, data_dict) context['connection'] = engine.connect() + timeout = context.get('query_timeout', 60000) _cache_types(context) try: # check if table existes + context['connection'].execute( + 'set local statement_timeout to {}'.format(timeout)) result = context['connection'].execute( 'select * from pg_tables where tablename = %s', data_dict['resource_id'] @@ -552,5 +561,11 @@ def search(context, data_dict): data_dict['resource_id']) }) return search_data(context, data_dict) + except Exception, e: + if 'due to statement timeout' in str(e): + raise p.toolkit.ValidationError({ + 'query': 'Search took too long' + }) + raise finally: context['connection'].close() From 3daacbea6712b448f47fe4342a85e077ad74466f Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 15 Aug 2012 17:21:24 +0100 Subject: [PATCH 060/130] [#2618] Prints to logging in html_resources --- ckan/html_resources/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index e11f9ee9da0..261ecdfe535 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -28,6 +28,7 @@ ''' import os.path import sys +import logging import ConfigParser from fanstatic import Library, Resource, Group, get_library_registry @@ -35,6 +36,8 @@ from ckan.include.rjsmin import jsmin from ckan.include.rcssmin import cssmin +log = logging.getLogger(__name__) + # TODO # loop through dirs to setup # warn on no entry point provided for fanstatic @@ -253,7 +256,7 @@ def minify(filename, resource_path, min_function): f = open(path_min, 'w') f.write(min_function(source)) f.close() - print 'minified %s' % path + log.info('minified %s' % path) def create_resource(path, lib_name, count): ''' create the fanstatic Resource ''' @@ -288,7 +291,7 @@ def create_resource(path, lib_name, count): resource = Resource(library, filename, **kw) # add the resource to this module fanstatic_name = '%s/%s' % (lib_name, filename) - print 'create resource %s' % fanstatic_name + log.info('create resource %s' % fanstatic_name) setattr(module, fanstatic_name, resource) return resource From 861c24a28d20ba36a5774a8a946452ceba70b86d Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 15 Aug 2012 17:25:10 +0100 Subject: [PATCH 061/130] [#2618] Add IE conditionals via config --- ckan/html_resources/__init__.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index 261ecdfe535..ce50ec3df04 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -281,9 +281,9 @@ def create_resource(path, lib_name, count): kw['dont_bundle'] = True kw['custom_order'] = count # FIXME needs resource.config options enabled - if False: - other_browsers = False - condition = '' + if path in IE_conditionals: + other_browsers = ('others' in IE_conditionals[path]) + condition = IE_conditionals[path][0] kw['renderer'] = IEConditionalRenderer( condition=condition, renderer=renderer, @@ -299,6 +299,7 @@ def create_resource(path, lib_name, count): dont_bundle = [] depends = {} groups = {} + IE_conditionals = {} # parse the resource.config file if it exists resource_path = os.path.dirname(__file__) @@ -317,6 +318,14 @@ def create_resource(path, lib_name, count): if config.has_section('groups'): items = config.items('groups') groups = dict((n, v.split()) for (n, v) in items) + if config.has_section('IE conditional'): + items = config.items('IE conditional') + for (n, v) in items: + files = v.split() + for f in files: + if f not in IE_conditionals: + IE_conditionals[f] = [] + IE_conditionals[f].append(n) library = Library(name, path) module = sys.modules[__name__] From 405ab3fa3348d8291f18140a0a305c9344974050 Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 15 Aug 2012 17:25:38 +0100 Subject: [PATCH 062/130] [#2618] dependencies for groups --- ckan/html_resources/__init__.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index ce50ec3df04..9bc0d8c70f7 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -275,7 +275,11 @@ def create_resource(path, lib_name, count): if path in depends: dependencies = [] for dependency in depends[path]: - dependencies.append(getattr(module, '%s/%s' % (lib_name, dependency))) + try: + res = getattr(module, '%s/%s' % (name, dependency)) + except AttributeError: + res = getattr(module, '%s' % dependency) + dependencies.append(res) kw['depends'] = dependencies if path in dont_bundle: kw['dont_bundle'] = True @@ -327,6 +331,16 @@ def create_resource(path, lib_name, count): IE_conditionals[f] = [] IE_conditionals[f].append(n) + + for group in groups: + if group in depends: + for resource in groups[group]: + if resource not in depends: + depends[resource] = [] + for dep in depends[group]: + if dep not in depends[resource]: + depends[resource].append(dep) + library = Library(name, path) module = sys.modules[__name__] @@ -366,6 +380,13 @@ def create_resource(path, lib_name, count): for member in groups[group_name]: fanstatic_name = '%s/%s' % (name, member) members.append(getattr(module, fanstatic_name)) + if group_name in depends: + for dependency in depends[group_name]: + try: + res = getattr(module, '%s/%s' % (name, dependency)) + except AttributeError: + res = getattr(module, '%s' % dependency) + members = [res] + members #.append(res) group = Group(members) fanstatic_name = '%s/%s' % (name, group_name) setattr(module, fanstatic_name, group) From 1c62a1e591f821d9fb3c450a60d4f695ce6cc754 Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 15 Aug 2012 17:26:35 +0100 Subject: [PATCH 063/130] [#2618] Upgrade templates --- ckan/html_resources/__init__.py | 9 ++++++--- ckan/templates/package/resource_read.html | 4 +++- ckan/templates/snippets/scripts.html | 5 ++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/ckan/html_resources/__init__.py b/ckan/html_resources/__init__.py index 9bc0d8c70f7..14159c74c63 100644 --- a/ckan/html_resources/__init__.py +++ b/ckan/html_resources/__init__.py @@ -396,12 +396,15 @@ def create_resource(path, lib_name, count): registry = get_library_registry() registry.add(library) -base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'public', 'base', 'javascript')) -create_library('base', base_path) - base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'public', 'base', 'vendor')) create_library('vendor', base_path) +base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'public', 'base', 'datapreview')) +create_library('datapreview', base_path) + +base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'public', 'base', 'javascript')) +create_library('base', base_path) + ### create our libraries here from any subdirectories ##for dirname, dirnames, filenames in os.walk(os.path.dirname(__file__)): ## if dirname == os.path.dirname(__file__): diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index 521c130bfaa..505bae2a1a4 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -130,6 +130,7 @@

Resource I {% block scripts %} {{ super() }} + {# @@ -145,7 +146,8 @@

Resource I - + #} + {% resource 'datapreview/datapreview' %} +{# #} +{# @@ -6,7 +7,9 @@ +#} +{% resource 'vendor/vendor' %} {# From 36667990f3255ce9f8da73e43ea2abbfa817e0ee Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 15 Aug 2012 17:28:00 +0100 Subject: [PATCH 064/130] [#2618] Update/add config files for resources --- ckan/public/base/datapreview/resource.config | 34 ++++++++++++++++++++ ckan/public/base/javascript/resource.config | 5 ++- ckan/public/base/vendor/resource.config | 28 ++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 ckan/public/base/datapreview/resource.config create mode 100644 ckan/public/base/vendor/resource.config diff --git a/ckan/public/base/datapreview/resource.config b/ckan/public/base/datapreview/resource.config new file mode 100644 index 00000000000..6388b4d0d74 --- /dev/null +++ b/ckan/public/base/datapreview/resource.config @@ -0,0 +1,34 @@ +[IE conditional] + +lte IE 8 = vendor/leaflet/0.3.1/leaflet.ie.css + +[depends] + +datapreview = vendor/jquery.js + +[groups] + +datapreview = + vendor/underscore/1.1.6/underscore.js + vendor/backbone/0.5.1/backbone.js + vendor/mustache/0.5.0-dev/mustache.js + vendor/bootstrap/2.0.3/bootstrap.js + vendor/jquery.mustache/jquery.mustache.js + + vendor/slickgrid/2.0.1/jquery-ui-1.8.16.custom.js + vendor/slickgrid/2.0.1/jquery.event.drag-2.0.js + vendor/slickgrid/2.0.1/slick.grid.js + vendor/flot/0.7/jquery.flot.js + vendor/moment/1.6.2/moment.js + vendor/leaflet/0.3.1/leaflet.js + + vendor/recline/recline.js + datapreview.js + + vendor/slickgrid/2.0.1/slick.grid.css + vendor/leaflet/0.3.1/leaflet.css + + vendor/leaflet/0.3.1/leaflet.ie.css + + vendor/recline/css/recline.css + css/datapreview.css diff --git a/ckan/public/base/javascript/resource.config b/ckan/public/base/javascript/resource.config index 16193be1218..3e3ef8b1653 100644 --- a/ckan/public/base/javascript/resource.config +++ b/ckan/public/base/javascript/resource.config @@ -1,3 +1,6 @@ +[depends] + +main = vendor/jquery.js [groups] @@ -19,7 +22,7 @@ main = modules/select-switch.js modules/slug-preview.js modules/basic-form.js - modules/confirm-delete.js + modules/confirm-action.js modules/api-info.js modules/autocomplete.js modules/custom-fields.js diff --git a/ckan/public/base/vendor/resource.config b/ckan/public/base/vendor/resource.config new file mode 100644 index 00000000000..ead36aee9a3 --- /dev/null +++ b/ckan/public/base/vendor/resource.config @@ -0,0 +1,28 @@ +[main] +dont_bundle = + jquery.js + +order = + jquery.js + jed.js + +[depends] + +vendor = jquery.js + +[groups] + +# jquery.js + +bootstrap = + bootstrap/js/bootstrap-transition.js + bootstrap/js/bootstrap-modal.js + bootstrap/js/bootstrap-alert.js + bootstrap/js/bootstrap-tab.js + bootstrap/js/bootstrap-button.js + +vendor = + jed.js + select2/select2.js + bootstrap + From d43d88cd5060e126541fee48515b715a2e3406ae Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 15 Aug 2012 17:30:14 +0100 Subject: [PATCH 065/130] [#2618] Add "unminified" js files --- .../vendor/bootstrap/2.0.3/bootstrap.js | 6 + .../2.0.1/jquery-ui-1.8.16.custom.js | 611 ++++++++++++++++++ .../slickgrid/2.0.1/jquery.event.drag-2.0.js | 6 + .../vendor/slickgrid/2.0.1/slick.grid.js | 84 +++ 4 files changed, 707 insertions(+) create mode 100644 ckan/public/base/datapreview/vendor/bootstrap/2.0.3/bootstrap.js create mode 100644 ckan/public/base/datapreview/vendor/slickgrid/2.0.1/jquery-ui-1.8.16.custom.js create mode 100644 ckan/public/base/datapreview/vendor/slickgrid/2.0.1/jquery.event.drag-2.0.js create mode 100644 ckan/public/base/datapreview/vendor/slickgrid/2.0.1/slick.grid.js diff --git a/ckan/public/base/datapreview/vendor/bootstrap/2.0.3/bootstrap.js b/ckan/public/base/datapreview/vendor/bootstrap/2.0.3/bootstrap.js new file mode 100644 index 00000000000..1f87730a3ae --- /dev/null +++ b/ckan/public/base/datapreview/vendor/bootstrap/2.0.3/bootstrap.js @@ -0,0 +1,6 @@ +/*! +* Bootstrap.js by @fat & @mdo +* Copyright 2012 Twitter, Inc. +* http://www.apache.org/licenses/LICENSE-2.0.txt +*/ +!function(a){a(function(){"use strict",a.support.transition=function(){var a=function(){var a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd",msTransition:"MSTransitionEnd",transition:"transitionend"},c;for(c in b)if(a.style[c]!==undefined)return b[c]}();return a&&{end:a}}()})}(window.jQuery),!function(a){"use strict";var b='[data-dismiss="alert"]',c=function(c){a(c).on("click",b,this.close)};c.prototype.close=function(b){function f(){e.trigger("closed").remove()}var c=a(this),d=c.attr("data-target"),e;d||(d=c.attr("href"),d=d&&d.replace(/.*(?=#[^\s]*$)/,"")),e=a(d),b&&b.preventDefault(),e.length||(e=c.hasClass("alert")?c:c.parent()),e.trigger(b=a.Event("close"));if(b.isDefaultPrevented())return;e.removeClass("in"),a.support.transition&&e.hasClass("fade")?e.on(a.support.transition.end,f):f()},a.fn.alert=function(b){return this.each(function(){var d=a(this),e=d.data("alert");e||d.data("alert",e=new c(this)),typeof b=="string"&&e[b].call(d)})},a.fn.alert.Constructor=c,a(function(){a("body").on("click.alert.data-api",b,c.prototype.close)})}(window.jQuery),!function(a){"use strict";var b=function(b,c){this.$element=a(b),this.options=a.extend({},a.fn.button.defaults,c)};b.prototype.setState=function(a){var b="disabled",c=this.$element,d=c.data(),e=c.is("input")?"val":"html";a+="Text",d.resetText||c.data("resetText",c[e]()),c[e](d[a]||this.options[a]),setTimeout(function(){a=="loadingText"?c.addClass(b).attr(b,b):c.removeClass(b).removeAttr(b)},0)},b.prototype.toggle=function(){var a=this.$element.parent('[data-toggle="buttons-radio"]');a&&a.find(".active").removeClass("active"),this.$element.toggleClass("active")},a.fn.button=function(c){return this.each(function(){var d=a(this),e=d.data("button"),f=typeof c=="object"&&c;e||d.data("button",e=new b(this,f)),c=="toggle"?e.toggle():c&&e.setState(c)})},a.fn.button.defaults={loadingText:"loading..."},a.fn.button.Constructor=b,a(function(){a("body").on("click.button.data-api","[data-toggle^=button]",function(b){var c=a(b.target);c.hasClass("btn")||(c=c.closest(".btn")),c.button("toggle")})})}(window.jQuery),!function(a){"use strict";var b=function(b,c){this.$element=a(b),this.options=c,this.options.slide&&this.slide(this.options.slide),this.options.pause=="hover"&&this.$element.on("mouseenter",a.proxy(this.pause,this)).on("mouseleave",a.proxy(this.cycle,this))};b.prototype={cycle:function(b){return b||(this.paused=!1),this.options.interval&&!this.paused&&(this.interval=setInterval(a.proxy(this.next,this),this.options.interval)),this},to:function(b){var c=this.$element.find(".active"),d=c.parent().children(),e=d.index(c),f=this;if(b>d.length-1||b<0)return;return this.sliding?this.$element.one("slid",function(){f.to(b)}):e==b?this.pause().cycle():this.slide(b>e?"next":"prev",a(d[b]))},pause:function(a){return a||(this.paused=!0),clearInterval(this.interval),this.interval=null,this},next:function(){if(this.sliding)return;return this.slide("next")},prev:function(){if(this.sliding)return;return this.slide("prev")},slide:function(b,c){var d=this.$element.find(".active"),e=c||d[b](),f=this.interval,g=b=="next"?"left":"right",h=b=="next"?"first":"last",i=this,j=a.Event("slide");this.sliding=!0,f&&this.pause(),e=e.length?e:this.$element.find(".item")[h]();if(e.hasClass("active"))return;if(a.support.transition&&this.$element.hasClass("slide")){this.$element.trigger(j);if(j.isDefaultPrevented())return;e.addClass(b),e[0].offsetWidth,d.addClass(g),e.addClass(g),this.$element.one(a.support.transition.end,function(){e.removeClass([b,g].join(" ")).addClass("active"),d.removeClass(["active",g].join(" ")),i.sliding=!1,setTimeout(function(){i.$element.trigger("slid")},0)})}else{this.$element.trigger(j);if(j.isDefaultPrevented())return;d.removeClass("active"),e.addClass("active"),this.sliding=!1,this.$element.trigger("slid")}return f&&this.cycle(),this}},a.fn.carousel=function(c){return this.each(function(){var d=a(this),e=d.data("carousel"),f=a.extend({},a.fn.carousel.defaults,typeof c=="object"&&c);e||d.data("carousel",e=new b(this,f)),typeof c=="number"?e.to(c):typeof c=="string"||(c=f.slide)?e[c]():f.interval&&e.cycle()})},a.fn.carousel.defaults={interval:5e3,pause:"hover"},a.fn.carousel.Constructor=b,a(function(){a("body").on("click.carousel.data-api","[data-slide]",function(b){var c=a(this),d,e=a(c.attr("data-target")||(d=c.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,"")),f=!e.data("modal")&&a.extend({},e.data(),c.data());e.carousel(f),b.preventDefault()})})}(window.jQuery),!function(a){"use strict";var b=function(b,c){this.$element=a(b),this.options=a.extend({},a.fn.collapse.defaults,c),this.options.parent&&(this.$parent=a(this.options.parent)),this.options.toggle&&this.toggle()};b.prototype={constructor:b,dimension:function(){var a=this.$element.hasClass("width");return a?"width":"height"},show:function(){var b,c,d,e;if(this.transitioning)return;b=this.dimension(),c=a.camelCase(["scroll",b].join("-")),d=this.$parent&&this.$parent.find("> .accordion-group > .in");if(d&&d.length){e=d.data("collapse");if(e&&e.transitioning)return;d.collapse("hide"),e||d.data("collapse",null)}this.$element[b](0),this.transition("addClass",a.Event("show"),"shown"),this.$element[b](this.$element[0][c])},hide:function(){var b;if(this.transitioning)return;b=this.dimension(),this.reset(this.$element[b]()),this.transition("removeClass",a.Event("hide"),"hidden"),this.$element[b](0)},reset:function(a){var b=this.dimension();return this.$element.removeClass("collapse")[b](a||"auto")[0].offsetWidth,this.$element[a!==null?"addClass":"removeClass"]("collapse"),this},transition:function(b,c,d){var e=this,f=function(){c.type=="show"&&e.reset(),e.transitioning=0,e.$element.trigger(d)};this.$element.trigger(c);if(c.isDefaultPrevented())return;this.transitioning=1,this.$element[b]("in"),a.support.transition&&this.$element.hasClass("collapse")?this.$element.one(a.support.transition.end,f):f()},toggle:function(){this[this.$element.hasClass("in")?"hide":"show"]()}},a.fn.collapse=function(c){return this.each(function(){var d=a(this),e=d.data("collapse"),f=typeof c=="object"&&c;e||d.data("collapse",e=new b(this,f)),typeof c=="string"&&e[c]()})},a.fn.collapse.defaults={toggle:!0},a.fn.collapse.Constructor=b,a(function(){a("body").on("click.collapse.data-api","[data-toggle=collapse]",function(b){var c=a(this),d,e=c.attr("data-target")||b.preventDefault()||(d=c.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,""),f=a(e).data("collapse")?"toggle":c.data();a(e).collapse(f)})})}(window.jQuery),!function(a){function d(){a(b).parent().removeClass("open")}"use strict";var b='[data-toggle="dropdown"]',c=function(b){var c=a(b).on("click.dropdown.data-api",this.toggle);a("html").on("click.dropdown.data-api",function(){c.parent().removeClass("open")})};c.prototype={constructor:c,toggle:function(b){var c=a(this),e,f,g;if(c.is(".disabled, :disabled"))return;return f=c.attr("data-target"),f||(f=c.attr("href"),f=f&&f.replace(/.*(?=#[^\s]*$)/,"")),e=a(f),e.length||(e=c.parent()),g=e.hasClass("open"),d(),g||e.toggleClass("open"),!1}},a.fn.dropdown=function(b){return this.each(function(){var d=a(this),e=d.data("dropdown");e||d.data("dropdown",e=new c(this)),typeof b=="string"&&e[b].call(d)})},a.fn.dropdown.Constructor=c,a(function(){a("html").on("click.dropdown.data-api",d),a("body").on("click.dropdown",".dropdown form",function(a){a.stopPropagation()}).on("click.dropdown.data-api",b,c.prototype.toggle)})}(window.jQuery),!function(a){function c(){var b=this,c=setTimeout(function(){b.$element.off(a.support.transition.end),d.call(b)},500);this.$element.one(a.support.transition.end,function(){clearTimeout(c),d.call(b)})}function d(a){this.$element.hide().trigger("hidden"),e.call(this)}function e(b){var c=this,d=this.$element.hasClass("fade")?"fade":"";if(this.isShown&&this.options.backdrop){var e=a.support.transition&&d;this.$backdrop=a('').css({position:this.element.css("position"),width:this.element.outerWidth(),height:this.element.outerHeight(), +top:this.element.css("top"),left:this.element.css("left")}));this.element=this.element.parent().data("resizable",this.element.data("resizable"));this.elementIsWrapper=true;this.element.css({marginLeft:this.originalElement.css("marginLeft"),marginTop:this.originalElement.css("marginTop"),marginRight:this.originalElement.css("marginRight"),marginBottom:this.originalElement.css("marginBottom")});this.originalElement.css({marginLeft:0,marginTop:0,marginRight:0,marginBottom:0});this.originalResizeStyle= +this.originalElement.css("resize");this.originalElement.css("resize","none");this._proportionallyResizeElements.push(this.originalElement.css({position:"static",zoom:1,display:"block"}));this.originalElement.css({margin:this.originalElement.css("margin")});this._proportionallyResize()}this.handles=a.handles||(!e(".ui-resizable-handle",this.element).length?"e,s,se":{n:".ui-resizable-n",e:".ui-resizable-e",s:".ui-resizable-s",w:".ui-resizable-w",se:".ui-resizable-se",sw:".ui-resizable-sw",ne:".ui-resizable-ne", +nw:".ui-resizable-nw"});if(this.handles.constructor==String){if(this.handles=="all")this.handles="n,e,s,w,se,sw,ne,nw";var c=this.handles.split(",");this.handles={};for(var d=0;d');/sw|se|ne|nw/.test(f)&&g.css({zIndex:++a.zIndex});"se"==f&&g.addClass("ui-icon ui-icon-gripsmall-diagonal-se");this.handles[f]=".ui-resizable-"+f;this.element.append(g)}}this._renderAxis=function(h){h=h||this.element;for(var i in this.handles){if(this.handles[i].constructor== +String)this.handles[i]=e(this.handles[i],this.element).show();if(this.elementIsWrapper&&this.originalElement[0].nodeName.match(/textarea|input|select|button/i)){var j=e(this.handles[i],this.element),l=0;l=/sw|ne|nw|se|n|s/.test(i)?j.outerHeight():j.outerWidth();j=["padding",/ne|nw|n/.test(i)?"Top":/se|sw|s/.test(i)?"Bottom":/^e$/.test(i)?"Right":"Left"].join("");h.css(j,l);this._proportionallyResize()}e(this.handles[i])}};this._renderAxis(this.element);this._handles=e(".ui-resizable-handle",this.element).disableSelection(); +this._handles.mouseover(function(){if(!b.resizing){if(this.className)var h=this.className.match(/ui-resizable-(se|sw|ne|nw|n|e|s|w)/i);b.axis=h&&h[1]?h[1]:"se"}});if(a.autoHide){this._handles.hide();e(this.element).addClass("ui-resizable-autohide").hover(function(){if(!a.disabled){e(this).removeClass("ui-resizable-autohide");b._handles.show()}},function(){if(!a.disabled)if(!b.resizing){e(this).addClass("ui-resizable-autohide");b._handles.hide()}})}this._mouseInit()},destroy:function(){this._mouseDestroy(); +var b=function(c){e(c).removeClass("ui-resizable ui-resizable-disabled ui-resizable-resizing").removeData("resizable").unbind(".resizable").find(".ui-resizable-handle").remove()};if(this.elementIsWrapper){b(this.element);var a=this.element;a.after(this.originalElement.css({position:a.css("position"),width:a.outerWidth(),height:a.outerHeight(),top:a.css("top"),left:a.css("left")})).remove()}this.originalElement.css("resize",this.originalResizeStyle);b(this.originalElement);return this},_mouseCapture:function(b){var a= +false;for(var c in this.handles)if(e(this.handles[c])[0]==b.target)a=true;return!this.options.disabled&&a},_mouseStart:function(b){var a=this.options,c=this.element.position(),d=this.element;this.resizing=true;this.documentScroll={top:e(document).scrollTop(),left:e(document).scrollLeft()};if(d.is(".ui-draggable")||/absolute/.test(d.css("position")))d.css({position:"absolute",top:c.top,left:c.left});e.browser.opera&&/relative/.test(d.css("position"))&&d.css({position:"relative",top:"auto",left:"auto"}); +this._renderProxy();c=m(this.helper.css("left"));var f=m(this.helper.css("top"));if(a.containment){c+=e(a.containment).scrollLeft()||0;f+=e(a.containment).scrollTop()||0}this.offset=this.helper.offset();this.position={left:c,top:f};this.size=this._helper?{width:d.outerWidth(),height:d.outerHeight()}:{width:d.width(),height:d.height()};this.originalSize=this._helper?{width:d.outerWidth(),height:d.outerHeight()}:{width:d.width(),height:d.height()};this.originalPosition={left:c,top:f};this.sizeDiff= +{width:d.outerWidth()-d.width(),height:d.outerHeight()-d.height()};this.originalMousePosition={left:b.pageX,top:b.pageY};this.aspectRatio=typeof a.aspectRatio=="number"?a.aspectRatio:this.originalSize.width/this.originalSize.height||1;a=e(".ui-resizable-"+this.axis).css("cursor");e("body").css("cursor",a=="auto"?this.axis+"-resize":a);d.addClass("ui-resizable-resizing");this._propagate("start",b);return true},_mouseDrag:function(b){var a=this.helper,c=this.originalMousePosition,d=this._change[this.axis]; +if(!d)return false;c=d.apply(this,[b,b.pageX-c.left||0,b.pageY-c.top||0]);this._updateVirtualBoundaries(b.shiftKey);if(this._aspectRatio||b.shiftKey)c=this._updateRatio(c,b);c=this._respectSize(c,b);this._propagate("resize",b);a.css({top:this.position.top+"px",left:this.position.left+"px",width:this.size.width+"px",height:this.size.height+"px"});!this._helper&&this._proportionallyResizeElements.length&&this._proportionallyResize();this._updateCache(c);this._trigger("resize",b,this.ui());return false}, +_mouseStop:function(b){this.resizing=false;var a=this.options,c=this;if(this._helper){var d=this._proportionallyResizeElements,f=d.length&&/textarea/i.test(d[0].nodeName);d=f&&e.ui.hasScroll(d[0],"left")?0:c.sizeDiff.height;f=f?0:c.sizeDiff.width;f={width:c.helper.width()-f,height:c.helper.height()-d};d=parseInt(c.element.css("left"),10)+(c.position.left-c.originalPosition.left)||null;var g=parseInt(c.element.css("top"),10)+(c.position.top-c.originalPosition.top)||null;a.animate||this.element.css(e.extend(f, +{top:g,left:d}));c.helper.height(c.size.height);c.helper.width(c.size.width);this._helper&&!a.animate&&this._proportionallyResize()}e("body").css("cursor","auto");this.element.removeClass("ui-resizable-resizing");this._propagate("stop",b);this._helper&&this.helper.remove();return false},_updateVirtualBoundaries:function(b){var a=this.options,c,d,f;a={minWidth:k(a.minWidth)?a.minWidth:0,maxWidth:k(a.maxWidth)?a.maxWidth:Infinity,minHeight:k(a.minHeight)?a.minHeight:0,maxHeight:k(a.maxHeight)?a.maxHeight: +Infinity};if(this._aspectRatio||b){b=a.minHeight*this.aspectRatio;d=a.minWidth/this.aspectRatio;c=a.maxHeight*this.aspectRatio;f=a.maxWidth/this.aspectRatio;if(b>a.minWidth)a.minWidth=b;if(d>a.minHeight)a.minHeight=d;if(cb.width,h=k(b.height)&&a.minHeight&&a.minHeight>b.height;if(g)b.width=a.minWidth;if(h)b.height=a.minHeight;if(d)b.width=a.maxWidth;if(f)b.height=a.maxHeight;var i=this.originalPosition.left+this.originalSize.width,j=this.position.top+this.size.height,l=/sw|nw|w/.test(c);c=/nw|ne|n/.test(c);if(g&&l)b.left=i-a.minWidth;if(d&&l)b.left=i-a.maxWidth;if(h&&c)b.top=j-a.minHeight;if(f&&c)b.top=j-a.maxHeight;if((a=!b.width&&!b.height)&&!b.left&&b.top)b.top=null;else if(a&&!b.top&&b.left)b.left= +null;return b},_proportionallyResize:function(){if(this._proportionallyResizeElements.length)for(var b=this.helper||this.element,a=0;a');var a=e.browser.msie&&e.browser.version<7,c=a?1:0;a=a?2:-1;this.helper.addClass(this._helper).css({width:this.element.outerWidth()+ +a,height:this.element.outerHeight()+a,position:"absolute",left:this.elementOffset.left-c+"px",top:this.elementOffset.top-c+"px",zIndex:++b.zIndex});this.helper.appendTo("body").disableSelection()}else this.helper=this.element},_change:{e:function(b,a){return{width:this.originalSize.width+a}},w:function(b,a){return{left:this.originalPosition.left+a,width:this.originalSize.width-a}},n:function(b,a,c){return{top:this.originalPosition.top+c,height:this.originalSize.height-c}},s:function(b,a,c){return{height:this.originalSize.height+ +c}},se:function(b,a,c){return e.extend(this._change.s.apply(this,arguments),this._change.e.apply(this,[b,a,c]))},sw:function(b,a,c){return e.extend(this._change.s.apply(this,arguments),this._change.w.apply(this,[b,a,c]))},ne:function(b,a,c){return e.extend(this._change.n.apply(this,arguments),this._change.e.apply(this,[b,a,c]))},nw:function(b,a,c){return e.extend(this._change.n.apply(this,arguments),this._change.w.apply(this,[b,a,c]))}},_propagate:function(b,a){e.ui.plugin.call(this,b,[a,this.ui()]); +b!="resize"&&this._trigger(b,a,this.ui())},plugins:{},ui:function(){return{originalElement:this.originalElement,element:this.element,helper:this.helper,position:this.position,size:this.size,originalSize:this.originalSize,originalPosition:this.originalPosition}}});e.extend(e.ui.resizable,{version:"1.8.16"});e.ui.plugin.add("resizable","alsoResize",{start:function(){var b=e(this).data("resizable").options,a=function(c){e(c).each(function(){var d=e(this);d.data("resizable-alsoresize",{width:parseInt(d.width(), +10),height:parseInt(d.height(),10),left:parseInt(d.css("left"),10),top:parseInt(d.css("top"),10),position:d.css("position")})})};if(typeof b.alsoResize=="object"&&!b.alsoResize.parentNode)if(b.alsoResize.length){b.alsoResize=b.alsoResize[0];a(b.alsoResize)}else e.each(b.alsoResize,function(c){a(c)});else a(b.alsoResize)},resize:function(b,a){var c=e(this).data("resizable");b=c.options;var d=c.originalSize,f=c.originalPosition,g={height:c.size.height-d.height||0,width:c.size.width-d.width||0,top:c.position.top- +f.top||0,left:c.position.left-f.left||0},h=function(i,j){e(i).each(function(){var l=e(this),q=e(this).data("resizable-alsoresize"),p={},r=j&&j.length?j:l.parents(a.originalElement[0]).length?["width","height"]:["width","height","top","left"];e.each(r,function(n,o){if((n=(q[o]||0)+(g[o]||0))&&n>=0)p[o]=n||null});if(e.browser.opera&&/relative/.test(l.css("position"))){c._revertToRelativePosition=true;l.css({position:"absolute",top:"auto",left:"auto"})}l.css(p)})};typeof b.alsoResize=="object"&&!b.alsoResize.nodeType? +e.each(b.alsoResize,function(i,j){h(i,j)}):h(b.alsoResize)},stop:function(){var b=e(this).data("resizable"),a=b.options,c=function(d){e(d).each(function(){var f=e(this);f.css({position:f.data("resizable-alsoresize").position})})};if(b._revertToRelativePosition){b._revertToRelativePosition=false;typeof a.alsoResize=="object"&&!a.alsoResize.nodeType?e.each(a.alsoResize,function(d){c(d)}):c(a.alsoResize)}e(this).removeData("resizable-alsoresize")}});e.ui.plugin.add("resizable","animate",{stop:function(b){var a= +e(this).data("resizable"),c=a.options,d=a._proportionallyResizeElements,f=d.length&&/textarea/i.test(d[0].nodeName),g=f&&e.ui.hasScroll(d[0],"left")?0:a.sizeDiff.height;f={width:a.size.width-(f?0:a.sizeDiff.width),height:a.size.height-g};g=parseInt(a.element.css("left"),10)+(a.position.left-a.originalPosition.left)||null;var h=parseInt(a.element.css("top"),10)+(a.position.top-a.originalPosition.top)||null;a.element.animate(e.extend(f,h&&g?{top:h,left:g}:{}),{duration:c.animateDuration,easing:c.animateEasing, +step:function(){var i={width:parseInt(a.element.css("width"),10),height:parseInt(a.element.css("height"),10),top:parseInt(a.element.css("top"),10),left:parseInt(a.element.css("left"),10)};d&&d.length&&e(d[0]).css({width:i.width,height:i.height});a._updateCache(i);a._propagate("resize",b)}})}});e.ui.plugin.add("resizable","containment",{start:function(){var b=e(this).data("resizable"),a=b.element,c=b.options.containment;if(a=c instanceof e?c.get(0):/parent/.test(c)?a.parent().get(0):c){b.containerElement= +e(a);if(/document/.test(c)||c==document){b.containerOffset={left:0,top:0};b.containerPosition={left:0,top:0};b.parentData={element:e(document),left:0,top:0,width:e(document).width(),height:e(document).height()||document.body.parentNode.scrollHeight}}else{var d=e(a),f=[];e(["Top","Right","Left","Bottom"]).each(function(i,j){f[i]=m(d.css("padding"+j))});b.containerOffset=d.offset();b.containerPosition=d.position();b.containerSize={height:d.innerHeight()-f[3],width:d.innerWidth()-f[1]};c=b.containerOffset; +var g=b.containerSize.height,h=b.containerSize.width;h=e.ui.hasScroll(a,"left")?a.scrollWidth:h;g=e.ui.hasScroll(a)?a.scrollHeight:g;b.parentData={element:a,left:c.left,top:c.top,width:h,height:g}}}},resize:function(b){var a=e(this).data("resizable"),c=a.options,d=a.containerOffset,f=a.position;b=a._aspectRatio||b.shiftKey;var g={top:0,left:0},h=a.containerElement;if(h[0]!=document&&/static/.test(h.css("position")))g=d;if(f.left<(a._helper?d.left:0)){a.size.width+=a._helper?a.position.left-d.left: +a.position.left-g.left;if(b)a.size.height=a.size.width/c.aspectRatio;a.position.left=c.helper?d.left:0}if(f.top<(a._helper?d.top:0)){a.size.height+=a._helper?a.position.top-d.top:a.position.top;if(b)a.size.width=a.size.height*c.aspectRatio;a.position.top=a._helper?d.top:0}a.offset.left=a.parentData.left+a.position.left;a.offset.top=a.parentData.top+a.position.top;c=Math.abs((a._helper?a.offset.left-g.left:a.offset.left-g.left)+a.sizeDiff.width);d=Math.abs((a._helper?a.offset.top-g.top:a.offset.top- +d.top)+a.sizeDiff.height);f=a.containerElement.get(0)==a.element.parent().get(0);g=/relative|absolute/.test(a.containerElement.css("position"));if(f&&g)c-=a.parentData.left;if(c+a.size.width>=a.parentData.width){a.size.width=a.parentData.width-c;if(b)a.size.height=a.size.width/a.aspectRatio}if(d+a.size.height>=a.parentData.height){a.size.height=a.parentData.height-d;if(b)a.size.width=a.size.height*a.aspectRatio}},stop:function(){var b=e(this).data("resizable"),a=b.options,c=b.containerOffset,d=b.containerPosition, +f=b.containerElement,g=e(b.helper),h=g.offset(),i=g.outerWidth()-b.sizeDiff.width;g=g.outerHeight()-b.sizeDiff.height;b._helper&&!a.animate&&/relative/.test(f.css("position"))&&e(this).css({left:h.left-d.left-c.left,width:i,height:g});b._helper&&!a.animate&&/static/.test(f.css("position"))&&e(this).css({left:h.left-d.left-c.left,width:i,height:g})}});e.ui.plugin.add("resizable","ghost",{start:function(){var b=e(this).data("resizable"),a=b.options,c=b.size;b.ghost=b.originalElement.clone();b.ghost.css({opacity:0.25, +display:"block",position:"relative",height:c.height,width:c.width,margin:0,left:0,top:0}).addClass("ui-resizable-ghost").addClass(typeof a.ghost=="string"?a.ghost:"");b.ghost.appendTo(b.helper)},resize:function(){var b=e(this).data("resizable");b.ghost&&b.ghost.css({position:"relative",height:b.size.height,width:b.size.width})},stop:function(){var b=e(this).data("resizable");b.ghost&&b.helper&&b.helper.get(0).removeChild(b.ghost.get(0))}});e.ui.plugin.add("resizable","grid",{resize:function(){var b= +e(this).data("resizable"),a=b.options,c=b.size,d=b.originalSize,f=b.originalPosition,g=b.axis;a.grid=typeof a.grid=="number"?[a.grid,a.grid]:a.grid;var h=Math.round((c.width-d.width)/(a.grid[0]||1))*(a.grid[0]||1);a=Math.round((c.height-d.height)/(a.grid[1]||1))*(a.grid[1]||1);if(/^(se|s|e)$/.test(g)){b.size.width=d.width+h;b.size.height=d.height+a}else if(/^(ne)$/.test(g)){b.size.width=d.width+h;b.size.height=d.height+a;b.position.top=f.top-a}else{if(/^(sw)$/.test(g)){b.size.width=d.width+h;b.size.height= +d.height+a}else{b.size.width=d.width+h;b.size.height=d.height+a;b.position.top=f.top-a}b.position.left=f.left-h}}});var m=function(b){return parseInt(b,10)||0},k=function(b){return!isNaN(parseInt(b,10))}})(jQuery); +;/* + * jQuery UI Selectable 1.8.16 + * + * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Selectables + * + * Depends: + * jquery.ui.core.js + * jquery.ui.mouse.js + * jquery.ui.widget.js + */ +(function(e){e.widget("ui.selectable",e.ui.mouse,{options:{appendTo:"body",autoRefresh:true,distance:0,filter:"*",tolerance:"touch"},_create:function(){var c=this;this.element.addClass("ui-selectable");this.dragged=false;var f;this.refresh=function(){f=e(c.options.filter,c.element[0]);f.each(function(){var d=e(this),b=d.offset();e.data(this,"selectable-item",{element:this,$element:d,left:b.left,top:b.top,right:b.left+d.outerWidth(),bottom:b.top+d.outerHeight(),startselected:false,selected:d.hasClass("ui-selected"), +selecting:d.hasClass("ui-selecting"),unselecting:d.hasClass("ui-unselecting")})})};this.refresh();this.selectees=f.addClass("ui-selectee");this._mouseInit();this.helper=e("
")},destroy:function(){this.selectees.removeClass("ui-selectee").removeData("selectable-item");this.element.removeClass("ui-selectable ui-selectable-disabled").removeData("selectable").unbind(".selectable");this._mouseDestroy();return this},_mouseStart:function(c){var f=this;this.opos=[c.pageX, +c.pageY];if(!this.options.disabled){var d=this.options;this.selectees=e(d.filter,this.element[0]);this._trigger("start",c);e(d.appendTo).append(this.helper);this.helper.css({left:c.clientX,top:c.clientY,width:0,height:0});d.autoRefresh&&this.refresh();this.selectees.filter(".ui-selected").each(function(){var b=e.data(this,"selectable-item");b.startselected=true;if(!c.metaKey){b.$element.removeClass("ui-selected");b.selected=false;b.$element.addClass("ui-unselecting");b.unselecting=true;f._trigger("unselecting", +c,{unselecting:b.element})}});e(c.target).parents().andSelf().each(function(){var b=e.data(this,"selectable-item");if(b){var g=!c.metaKey||!b.$element.hasClass("ui-selected");b.$element.removeClass(g?"ui-unselecting":"ui-selected").addClass(g?"ui-selecting":"ui-unselecting");b.unselecting=!g;b.selecting=g;(b.selected=g)?f._trigger("selecting",c,{selecting:b.element}):f._trigger("unselecting",c,{unselecting:b.element});return false}})}},_mouseDrag:function(c){var f=this;this.dragged=true;if(!this.options.disabled){var d= +this.options,b=this.opos[0],g=this.opos[1],h=c.pageX,i=c.pageY;if(b>h){var j=h;h=b;b=j}if(g>i){j=i;i=g;g=j}this.helper.css({left:b,top:g,width:h-b,height:i-g});this.selectees.each(function(){var a=e.data(this,"selectable-item");if(!(!a||a.element==f.element[0])){var k=false;if(d.tolerance=="touch")k=!(a.left>h||a.righti||a.bottomb&&a.rightg&&a.bottom *",opacity:false,placeholder:false,revert:false,scroll:true,scrollSensitivity:20,scrollSpeed:20,scope:"default",tolerance:"intersect",zIndex:1E3},_create:function(){var a=this.options;this.containerCache={};this.element.addClass("ui-sortable"); +this.refresh();this.floating=this.items.length?a.axis==="x"||/left|right/.test(this.items[0].item.css("float"))||/inline|table-cell/.test(this.items[0].item.css("display")):false;this.offset=this.element.offset();this._mouseInit()},destroy:function(){this.element.removeClass("ui-sortable ui-sortable-disabled").removeData("sortable").unbind(".sortable");this._mouseDestroy();for(var a=this.items.length-1;a>=0;a--)this.items[a].item.removeData("sortable-item");return this},_setOption:function(a,b){if(a=== +"disabled"){this.options[a]=b;this.widget()[b?"addClass":"removeClass"]("ui-sortable-disabled")}else d.Widget.prototype._setOption.apply(this,arguments)},_mouseCapture:function(a,b){if(this.reverting)return false;if(this.options.disabled||this.options.type=="static")return false;this._refreshItems(a);var c=null,e=this;d(a.target).parents().each(function(){if(d.data(this,"sortable-item")==e){c=d(this);return false}});if(d.data(a.target,"sortable-item")==e)c=d(a.target);if(!c)return false;if(this.options.handle&& +!b){var f=false;d(this.options.handle,c).find("*").andSelf().each(function(){if(this==a.target)f=true});if(!f)return false}this.currentItem=c;this._removeCurrentsFromItems();return true},_mouseStart:function(a,b,c){b=this.options;var e=this;this.currentContainer=this;this.refreshPositions();this.helper=this._createHelper(a);this._cacheHelperProportions();this._cacheMargins();this.scrollParent=this.helper.scrollParent();this.offset=this.currentItem.offset();this.offset={top:this.offset.top-this.margins.top, +left:this.offset.left-this.margins.left};this.helper.css("position","absolute");this.cssPosition=this.helper.css("position");d.extend(this.offset,{click:{left:a.pageX-this.offset.left,top:a.pageY-this.offset.top},parent:this._getParentOffset(),relative:this._getRelativeOffset()});this.originalPosition=this._generatePosition(a);this.originalPageX=a.pageX;this.originalPageY=a.pageY;b.cursorAt&&this._adjustOffsetFromHelper(b.cursorAt);this.domPosition={prev:this.currentItem.prev()[0],parent:this.currentItem.parent()[0]}; +this.helper[0]!=this.currentItem[0]&&this.currentItem.hide();this._createPlaceholder();b.containment&&this._setContainment();if(b.cursor){if(d("body").css("cursor"))this._storedCursor=d("body").css("cursor");d("body").css("cursor",b.cursor)}if(b.opacity){if(this.helper.css("opacity"))this._storedOpacity=this.helper.css("opacity");this.helper.css("opacity",b.opacity)}if(b.zIndex){if(this.helper.css("zIndex"))this._storedZIndex=this.helper.css("zIndex");this.helper.css("zIndex",b.zIndex)}if(this.scrollParent[0]!= +document&&this.scrollParent[0].tagName!="HTML")this.overflowOffset=this.scrollParent.offset();this._trigger("start",a,this._uiHash());this._preserveHelperProportions||this._cacheHelperProportions();if(!c)for(c=this.containers.length-1;c>=0;c--)this.containers[c]._trigger("activate",a,e._uiHash(this));if(d.ui.ddmanager)d.ui.ddmanager.current=this;d.ui.ddmanager&&!b.dropBehaviour&&d.ui.ddmanager.prepareOffsets(this,a);this.dragging=true;this.helper.addClass("ui-sortable-helper");this._mouseDrag(a); +return true},_mouseDrag:function(a){this.position=this._generatePosition(a);this.positionAbs=this._convertPositionTo("absolute");if(!this.lastPositionAbs)this.lastPositionAbs=this.positionAbs;if(this.options.scroll){var b=this.options,c=false;if(this.scrollParent[0]!=document&&this.scrollParent[0].tagName!="HTML"){if(this.overflowOffset.top+this.scrollParent[0].offsetHeight-a.pageY=0;b--){c=this.items[b];var e=c.item[0],f=this._intersectsWithPointer(c);if(f)if(e!=this.currentItem[0]&&this.placeholder[f==1?"next":"prev"]()[0]!=e&&!d.ui.contains(this.placeholder[0],e)&&(this.options.type=="semi-dynamic"?!d.ui.contains(this.element[0], +e):true)){this.direction=f==1?"down":"up";if(this.options.tolerance=="pointer"||this._intersectsWithSides(c))this._rearrange(a,c);else break;this._trigger("change",a,this._uiHash());break}}this._contactContainers(a);d.ui.ddmanager&&d.ui.ddmanager.drag(this,a);this._trigger("sort",a,this._uiHash());this.lastPositionAbs=this.positionAbs;return false},_mouseStop:function(a,b){if(a){d.ui.ddmanager&&!this.options.dropBehaviour&&d.ui.ddmanager.drop(this,a);if(this.options.revert){var c=this;b=c.placeholder.offset(); +c.reverting=true;d(this.helper).animate({left:b.left-this.offset.parent.left-c.margins.left+(this.offsetParent[0]==document.body?0:this.offsetParent[0].scrollLeft),top:b.top-this.offset.parent.top-c.margins.top+(this.offsetParent[0]==document.body?0:this.offsetParent[0].scrollTop)},parseInt(this.options.revert,10)||500,function(){c._clear(a)})}else this._clear(a,b);return false}},cancel:function(){var a=this;if(this.dragging){this._mouseUp({target:null});this.options.helper=="original"?this.currentItem.css(this._storedCSS).removeClass("ui-sortable-helper"): +this.currentItem.show();for(var b=this.containers.length-1;b>=0;b--){this.containers[b]._trigger("deactivate",null,a._uiHash(this));if(this.containers[b].containerCache.over){this.containers[b]._trigger("out",null,a._uiHash(this));this.containers[b].containerCache.over=0}}}if(this.placeholder){this.placeholder[0].parentNode&&this.placeholder[0].parentNode.removeChild(this.placeholder[0]);this.options.helper!="original"&&this.helper&&this.helper[0].parentNode&&this.helper.remove();d.extend(this,{helper:null, +dragging:false,reverting:false,_noFinalSort:null});this.domPosition.prev?d(this.domPosition.prev).after(this.currentItem):d(this.domPosition.parent).prepend(this.currentItem)}return this},serialize:function(a){var b=this._getItemsAsjQuery(a&&a.connected),c=[];a=a||{};d(b).each(function(){var e=(d(a.item||this).attr(a.attribute||"id")||"").match(a.expression||/(.+)[-=_](.+)/);if(e)c.push((a.key||e[1]+"[]")+"="+(a.key&&a.expression?e[1]:e[2]))});!c.length&&a.key&&c.push(a.key+"=");return c.join("&")}, +toArray:function(a){var b=this._getItemsAsjQuery(a&&a.connected),c=[];a=a||{};b.each(function(){c.push(d(a.item||this).attr(a.attribute||"id")||"")});return c},_intersectsWith:function(a){var b=this.positionAbs.left,c=b+this.helperProportions.width,e=this.positionAbs.top,f=e+this.helperProportions.height,g=a.left,h=g+a.width,i=a.top,k=i+a.height,j=this.offset.click.top,l=this.offset.click.left;j=e+j>i&&e+jg&&b+la[this.floating?"width":"height"]?j:g0?"down":"up")},_getDragHorizontalDirection:function(){var a=this.positionAbs.left-this.lastPositionAbs.left;return a!=0&&(a>0?"right":"left")},refresh:function(a){this._refreshItems(a);this.refreshPositions();return this},_connectWith:function(){var a=this.options;return a.connectWith.constructor==String?[a.connectWith]:a.connectWith},_getItemsAsjQuery:function(a){var b=[],c=[],e=this._connectWith(); +if(e&&a)for(a=e.length-1;a>=0;a--)for(var f=d(e[a]),g=f.length-1;g>=0;g--){var h=d.data(f[g],"sortable");if(h&&h!=this&&!h.options.disabled)c.push([d.isFunction(h.options.items)?h.options.items.call(h.element):d(h.options.items,h.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"),h])}c.push([d.isFunction(this.options.items)?this.options.items.call(this.element,null,{options:this.options,item:this.currentItem}):d(this.options.items,this.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"), +this]);for(a=c.length-1;a>=0;a--)c[a][0].each(function(){b.push(this)});return d(b)},_removeCurrentsFromItems:function(){for(var a=this.currentItem.find(":data(sortable-item)"),b=0;b=0;f--)for(var g=d(e[f]),h=g.length-1;h>=0;h--){var i=d.data(g[h],"sortable");if(i&&i!=this&&!i.options.disabled){c.push([d.isFunction(i.options.items)?i.options.items.call(i.element[0],a,{item:this.currentItem}):d(i.options.items,i.element),i]);this.containers.push(i)}}for(f=c.length-1;f>=0;f--){a=c[f][1];e=c[f][0];h=0;for(g=e.length;h=0;b--){var c=this.items[b];if(!(c.instance!=this.currentContainer&&this.currentContainer&&c.item[0]!=this.currentItem[0])){var e=this.options.toleranceElement?d(this.options.toleranceElement,c.item):c.item;if(!a){c.width=e.outerWidth();c.height=e.outerHeight()}e=e.offset();c.left=e.left;c.top=e.top}}if(this.options.custom&&this.options.custom.refreshContainers)this.options.custom.refreshContainers.call(this);else for(b= +this.containers.length-1;b>=0;b--){e=this.containers[b].element.offset();this.containers[b].containerCache.left=e.left;this.containers[b].containerCache.top=e.top;this.containers[b].containerCache.width=this.containers[b].element.outerWidth();this.containers[b].containerCache.height=this.containers[b].element.outerHeight()}return this},_createPlaceholder:function(a){var b=a||this,c=b.options;if(!c.placeholder||c.placeholder.constructor==String){var e=c.placeholder;c.placeholder={element:function(){var f= +d(document.createElement(b.currentItem[0].nodeName)).addClass(e||b.currentItem[0].className+" ui-sortable-placeholder").removeClass("ui-sortable-helper")[0];if(!e)f.style.visibility="hidden";return f},update:function(f,g){if(!(e&&!c.forcePlaceholderSize)){g.height()||g.height(b.currentItem.innerHeight()-parseInt(b.currentItem.css("paddingTop")||0,10)-parseInt(b.currentItem.css("paddingBottom")||0,10));g.width()||g.width(b.currentItem.innerWidth()-parseInt(b.currentItem.css("paddingLeft")||0,10)-parseInt(b.currentItem.css("paddingRight")|| +0,10))}}}}b.placeholder=d(c.placeholder.element.call(b.element,b.currentItem));b.currentItem.after(b.placeholder);c.placeholder.update(b,b.placeholder)},_contactContainers:function(a){for(var b=null,c=null,e=this.containers.length-1;e>=0;e--)if(!d.ui.contains(this.currentItem[0],this.containers[e].element[0]))if(this._intersectsWith(this.containers[e].containerCache)){if(!(b&&d.ui.contains(this.containers[e].element[0],b.element[0]))){b=this.containers[e];c=e}}else if(this.containers[e].containerCache.over){this.containers[e]._trigger("out", +a,this._uiHash(this));this.containers[e].containerCache.over=0}if(b)if(this.containers.length===1){this.containers[c]._trigger("over",a,this._uiHash(this));this.containers[c].containerCache.over=1}else if(this.currentContainer!=this.containers[c]){b=1E4;e=null;for(var f=this.positionAbs[this.containers[c].floating?"left":"top"],g=this.items.length-1;g>=0;g--)if(d.ui.contains(this.containers[c].element[0],this.items[g].item[0])){var h=this.items[g][this.containers[c].floating?"left":"top"];if(Math.abs(h- +f)this.containment[2])f=this.containment[2]+this.offset.click.left;if(a.pageY-this.offset.click.top>this.containment[3])g=this.containment[3]+this.offset.click.top}if(b.grid){g=this.originalPageY+Math.round((g- +this.originalPageY)/b.grid[1])*b.grid[1];g=this.containment?!(g-this.offset.click.topthis.containment[3])?g:!(g-this.offset.click.topthis.containment[2])?f:!(f-this.offset.click.left=0;e--)if(d.ui.contains(this.containers[e].element[0],this.currentItem[0])&&!b){c.push(function(f){return function(g){f._trigger("receive",g,this._uiHash(this))}}.call(this,this.containers[e]));c.push(function(f){return function(g){f._trigger("update",g,this._uiHash(this))}}.call(this,this.containers[e]))}}for(e=this.containers.length-1;e>=0;e--){b||c.push(function(f){return function(g){f._trigger("deactivate",g,this._uiHash(this))}}.call(this, +this.containers[e]));if(this.containers[e].containerCache.over){c.push(function(f){return function(g){f._trigger("out",g,this._uiHash(this))}}.call(this,this.containers[e]));this.containers[e].containerCache.over=0}}this._storedCursor&&d("body").css("cursor",this._storedCursor);this._storedOpacity&&this.helper.css("opacity",this._storedOpacity);if(this._storedZIndex)this.helper.css("zIndex",this._storedZIndex=="auto"?"":this._storedZIndex);this.dragging=false;if(this.cancelHelperRemoval){if(!b){this._trigger("beforeStop", +a,this._uiHash());for(e=0;e").appendTo(this.element).addClass("ui-slider-range ui-widget-header"+(b.range==="min"||b.range==="max"?" ui-slider-range-"+b.range:""))}for(var j=c.length;j"); +this.handles=c.add(d(e.join("")).appendTo(a.element));this.handle=this.handles.eq(0);this.handles.add(this.range).filter("a").click(function(g){g.preventDefault()}).hover(function(){b.disabled||d(this).addClass("ui-state-hover")},function(){d(this).removeClass("ui-state-hover")}).focus(function(){if(b.disabled)d(this).blur();else{d(".ui-slider .ui-state-focus").removeClass("ui-state-focus");d(this).addClass("ui-state-focus")}}).blur(function(){d(this).removeClass("ui-state-focus")});this.handles.each(function(g){d(this).data("index.ui-slider-handle", +g)});this.handles.keydown(function(g){var k=true,l=d(this).data("index.ui-slider-handle"),i,h,m;if(!a.options.disabled){switch(g.keyCode){case d.ui.keyCode.HOME:case d.ui.keyCode.END:case d.ui.keyCode.PAGE_UP:case d.ui.keyCode.PAGE_DOWN:case d.ui.keyCode.UP:case d.ui.keyCode.RIGHT:case d.ui.keyCode.DOWN:case d.ui.keyCode.LEFT:k=false;if(!a._keySliding){a._keySliding=true;d(this).addClass("ui-state-active");i=a._start(g,l);if(i===false)return}break}m=a.options.step;i=a.options.values&&a.options.values.length? +(h=a.values(l)):(h=a.value());switch(g.keyCode){case d.ui.keyCode.HOME:h=a._valueMin();break;case d.ui.keyCode.END:h=a._valueMax();break;case d.ui.keyCode.PAGE_UP:h=a._trimAlignValue(i+(a._valueMax()-a._valueMin())/5);break;case d.ui.keyCode.PAGE_DOWN:h=a._trimAlignValue(i-(a._valueMax()-a._valueMin())/5);break;case d.ui.keyCode.UP:case d.ui.keyCode.RIGHT:if(i===a._valueMax())return;h=a._trimAlignValue(i+m);break;case d.ui.keyCode.DOWN:case d.ui.keyCode.LEFT:if(i===a._valueMin())return;h=a._trimAlignValue(i- +m);break}a._slide(g,l,h);return k}}).keyup(function(g){var k=d(this).data("index.ui-slider-handle");if(a._keySliding){a._keySliding=false;a._stop(g,k);a._change(g,k);d(this).removeClass("ui-state-active")}});this._refreshValue();this._animateOff=false},destroy:function(){this.handles.remove();this.range.remove();this.element.removeClass("ui-slider ui-slider-horizontal ui-slider-vertical ui-slider-disabled ui-widget ui-widget-content ui-corner-all").removeData("slider").unbind(".slider");this._mouseDestroy(); +return this},_mouseCapture:function(a){var b=this.options,c,f,e,j,g;if(b.disabled)return false;this.elementSize={width:this.element.outerWidth(),height:this.element.outerHeight()};this.elementOffset=this.element.offset();c=this._normValueFromMouse({x:a.pageX,y:a.pageY});f=this._valueMax()-this._valueMin()+1;j=this;this.handles.each(function(k){var l=Math.abs(c-j.values(k));if(f>l){f=l;e=d(this);g=k}});if(b.range===true&&this.values(1)===b.min){g+=1;e=d(this.handles[g])}if(this._start(a,g)===false)return false; +this._mouseSliding=true;j._handleIndex=g;e.addClass("ui-state-active").focus();b=e.offset();this._clickOffset=!d(a.target).parents().andSelf().is(".ui-slider-handle")?{left:0,top:0}:{left:a.pageX-b.left-e.width()/2,top:a.pageY-b.top-e.height()/2-(parseInt(e.css("borderTopWidth"),10)||0)-(parseInt(e.css("borderBottomWidth"),10)||0)+(parseInt(e.css("marginTop"),10)||0)};this.handles.hasClass("ui-state-hover")||this._slide(a,g,c);return this._animateOff=true},_mouseStart:function(){return true},_mouseDrag:function(a){var b= +this._normValueFromMouse({x:a.pageX,y:a.pageY});this._slide(a,this._handleIndex,b);return false},_mouseStop:function(a){this.handles.removeClass("ui-state-active");this._mouseSliding=false;this._stop(a,this._handleIndex);this._change(a,this._handleIndex);this._clickOffset=this._handleIndex=null;return this._animateOff=false},_detectOrientation:function(){this.orientation=this.options.orientation==="vertical"?"vertical":"horizontal"},_normValueFromMouse:function(a){var b;if(this.orientation==="horizontal"){b= +this.elementSize.width;a=a.x-this.elementOffset.left-(this._clickOffset?this._clickOffset.left:0)}else{b=this.elementSize.height;a=a.y-this.elementOffset.top-(this._clickOffset?this._clickOffset.top:0)}b=a/b;if(b>1)b=1;if(b<0)b=0;if(this.orientation==="vertical")b=1-b;a=this._valueMax()-this._valueMin();return this._trimAlignValue(this._valueMin()+b*a)},_start:function(a,b){var c={handle:this.handles[b],value:this.value()};if(this.options.values&&this.options.values.length){c.value=this.values(b); +c.values=this.values()}return this._trigger("start",a,c)},_slide:function(a,b,c){var f;if(this.options.values&&this.options.values.length){f=this.values(b?0:1);if(this.options.values.length===2&&this.options.range===true&&(b===0&&c>f||b===1&&c1){this.options.values[a]=this._trimAlignValue(b);this._refreshValue();this._change(null,a)}else if(arguments.length)if(d.isArray(arguments[0])){c=this.options.values;f=arguments[0];for(e=0;e=this._valueMax())return this._valueMax();var b=this.options.step>0?this.options.step:1,c=(a-this._valueMin())%b;a=a-c;if(Math.abs(c)*2>=b)a+=c>0?b:-b;return parseFloat(a.toFixed(5))},_valueMin:function(){return this.options.min},_valueMax:function(){return this.options.max},_refreshValue:function(){var a= +this.options.range,b=this.options,c=this,f=!this._animateOff?b.animate:false,e,j={},g,k,l,i;if(this.options.values&&this.options.values.length)this.handles.each(function(h){e=(c.values(h)-c._valueMin())/(c._valueMax()-c._valueMin())*100;j[c.orientation==="horizontal"?"left":"bottom"]=e+"%";d(this).stop(1,1)[f?"animate":"css"](j,b.animate);if(c.options.range===true)if(c.orientation==="horizontal"){if(h===0)c.range.stop(1,1)[f?"animate":"css"]({left:e+"%"},b.animate);if(h===1)c.range[f?"animate":"css"]({width:e- +g+"%"},{queue:false,duration:b.animate})}else{if(h===0)c.range.stop(1,1)[f?"animate":"css"]({bottom:e+"%"},b.animate);if(h===1)c.range[f?"animate":"css"]({height:e-g+"%"},{queue:false,duration:b.animate})}g=e});else{k=this.value();l=this._valueMin();i=this._valueMax();e=i!==l?(k-l)/(i-l)*100:0;j[c.orientation==="horizontal"?"left":"bottom"]=e+"%";this.handle.stop(1,1)[f?"animate":"css"](j,b.animate);if(a==="min"&&this.orientation==="horizontal")this.range.stop(1,1)[f?"animate":"css"]({width:e+"%"}, +b.animate);if(a==="max"&&this.orientation==="horizontal")this.range[f?"animate":"css"]({width:100-e+"%"},{queue:false,duration:b.animate});if(a==="min"&&this.orientation==="vertical")this.range.stop(1,1)[f?"animate":"css"]({height:e+"%"},b.animate);if(a==="max"&&this.orientation==="vertical")this.range[f?"animate":"css"]({height:100-e+"%"},{queue:false,duration:b.animate})}}});d.extend(d.ui.slider,{version:"1.8.16"})})(jQuery); +;/* + * jQuery UI Datepicker 1.8.16 + * + * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Datepicker + * + * Depends: + * jquery.ui.core.js + */ +(function(d,C){function M(){this.debug=false;this._curInst=null;this._keyEvent=false;this._disabledInputs=[];this._inDialog=this._datepickerShowing=false;this._mainDivId="ui-datepicker-div";this._inlineClass="ui-datepicker-inline";this._appendClass="ui-datepicker-append";this._triggerClass="ui-datepicker-trigger";this._dialogClass="ui-datepicker-dialog";this._disableClass="ui-datepicker-disabled";this._unselectableClass="ui-datepicker-unselectable";this._currentClass="ui-datepicker-current-day";this._dayOverClass= +"ui-datepicker-days-cell-over";this.regional=[];this.regional[""]={closeText:"Done",prevText:"Prev",nextText:"Next",currentText:"Today",monthNames:["January","February","March","April","May","June","July","August","September","October","November","December"],monthNamesShort:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"],dayNames:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],dayNamesShort:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],dayNamesMin:["Su", +"Mo","Tu","We","Th","Fr","Sa"],weekHeader:"Wk",dateFormat:"mm/dd/yy",firstDay:0,isRTL:false,showMonthAfterYear:false,yearSuffix:""};this._defaults={showOn:"focus",showAnim:"fadeIn",showOptions:{},defaultDate:null,appendText:"",buttonText:"...",buttonImage:"",buttonImageOnly:false,hideIfNoPrevNext:false,navigationAsDateFormat:false,gotoCurrent:false,changeMonth:false,changeYear:false,yearRange:"c-10:c+10",showOtherMonths:false,selectOtherMonths:false,showWeek:false,calculateWeek:this.iso8601Week,shortYearCutoff:"+10", +minDate:null,maxDate:null,duration:"fast",beforeShowDay:null,beforeShow:null,onSelect:null,onChangeMonthYear:null,onClose:null,numberOfMonths:1,showCurrentAtPos:0,stepMonths:1,stepBigMonths:12,altField:"",altFormat:"",constrainInput:true,showButtonPanel:false,autoSize:false,disabled:false};d.extend(this._defaults,this.regional[""]);this.dpDiv=N(d('
'))}function N(a){return a.bind("mouseout", +function(b){b=d(b.target).closest("button, .ui-datepicker-prev, .ui-datepicker-next, .ui-datepicker-calendar td a");b.length&&b.removeClass("ui-state-hover ui-datepicker-prev-hover ui-datepicker-next-hover")}).bind("mouseover",function(b){b=d(b.target).closest("button, .ui-datepicker-prev, .ui-datepicker-next, .ui-datepicker-calendar td a");if(!(d.datepicker._isDisabledDatepicker(J.inline?a.parent()[0]:J.input[0])||!b.length)){b.parents(".ui-datepicker-calendar").find("a").removeClass("ui-state-hover"); +b.addClass("ui-state-hover");b.hasClass("ui-datepicker-prev")&&b.addClass("ui-datepicker-prev-hover");b.hasClass("ui-datepicker-next")&&b.addClass("ui-datepicker-next-hover")}})}function H(a,b){d.extend(a,b);for(var c in b)if(b[c]==null||b[c]==C)a[c]=b[c];return a}d.extend(d.ui,{datepicker:{version:"1.8.16"}});var B=(new Date).getTime(),J;d.extend(M.prototype,{markerClassName:"hasDatepicker",maxRows:4,log:function(){this.debug&&console.log.apply("",arguments)},_widgetDatepicker:function(){return this.dpDiv}, +setDefaults:function(a){H(this._defaults,a||{});return this},_attachDatepicker:function(a,b){var c=null;for(var e in this._defaults){var f=a.getAttribute("date:"+e);if(f){c=c||{};try{c[e]=eval(f)}catch(h){c[e]=f}}}e=a.nodeName.toLowerCase();f=e=="div"||e=="span";if(!a.id){this.uuid+=1;a.id="dp"+this.uuid}var i=this._newInst(d(a),f);i.settings=d.extend({},b||{},c||{});if(e=="input")this._connectDatepicker(a,i);else f&&this._inlineDatepicker(a,i)},_newInst:function(a,b){return{id:a[0].id.replace(/([^A-Za-z0-9_-])/g, +"\\\\$1"),input:a,selectedDay:0,selectedMonth:0,selectedYear:0,drawMonth:0,drawYear:0,inline:b,dpDiv:!b?this.dpDiv:N(d('
'))}},_connectDatepicker:function(a,b){var c=d(a);b.append=d([]);b.trigger=d([]);if(!c.hasClass(this.markerClassName)){this._attachments(c,b);c.addClass(this.markerClassName).keydown(this._doKeyDown).keypress(this._doKeyPress).keyup(this._doKeyUp).bind("setData.datepicker", +function(e,f,h){b.settings[f]=h}).bind("getData.datepicker",function(e,f){return this._get(b,f)});this._autoSize(b);d.data(a,"datepicker",b);b.settings.disabled&&this._disableDatepicker(a)}},_attachments:function(a,b){var c=this._get(b,"appendText"),e=this._get(b,"isRTL");b.append&&b.append.remove();if(c){b.append=d(''+c+"");a[e?"before":"after"](b.append)}a.unbind("focus",this._showDatepicker);b.trigger&&b.trigger.remove();c=this._get(b,"showOn");if(c== +"focus"||c=="both")a.focus(this._showDatepicker);if(c=="button"||c=="both"){c=this._get(b,"buttonText");var f=this._get(b,"buttonImage");b.trigger=d(this._get(b,"buttonImageOnly")?d("").addClass(this._triggerClass).attr({src:f,alt:c,title:c}):d('').addClass(this._triggerClass).html(f==""?c:d("").attr({src:f,alt:c,title:c})));a[e?"before":"after"](b.trigger);b.trigger.click(function(){d.datepicker._datepickerShowing&&d.datepicker._lastInput==a[0]?d.datepicker._hideDatepicker(): +d.datepicker._showDatepicker(a[0]);return false})}},_autoSize:function(a){if(this._get(a,"autoSize")&&!a.inline){var b=new Date(2009,11,20),c=this._get(a,"dateFormat");if(c.match(/[DM]/)){var e=function(f){for(var h=0,i=0,g=0;gh){h=f[g].length;i=g}return i};b.setMonth(e(this._get(a,c.match(/MM/)?"monthNames":"monthNamesShort")));b.setDate(e(this._get(a,c.match(/DD/)?"dayNames":"dayNamesShort"))+20-b.getDay())}a.input.attr("size",this._formatDate(a,b).length)}},_inlineDatepicker:function(a, +b){var c=d(a);if(!c.hasClass(this.markerClassName)){c.addClass(this.markerClassName).append(b.dpDiv).bind("setData.datepicker",function(e,f,h){b.settings[f]=h}).bind("getData.datepicker",function(e,f){return this._get(b,f)});d.data(a,"datepicker",b);this._setDate(b,this._getDefaultDate(b),true);this._updateDatepicker(b);this._updateAlternate(b);b.settings.disabled&&this._disableDatepicker(a);b.dpDiv.css("display","block")}},_dialogDatepicker:function(a,b,c,e,f){a=this._dialogInst;if(!a){this.uuid+= +1;this._dialogInput=d('');this._dialogInput.keydown(this._doKeyDown);d("body").append(this._dialogInput);a=this._dialogInst=this._newInst(this._dialogInput,false);a.settings={};d.data(this._dialogInput[0],"datepicker",a)}H(a.settings,e||{});b=b&&b.constructor==Date?this._formatDate(a,b):b;this._dialogInput.val(b);this._pos=f?f.length?f:[f.pageX,f.pageY]:null;if(!this._pos)this._pos=[document.documentElement.clientWidth/ +2-100+(document.documentElement.scrollLeft||document.body.scrollLeft),document.documentElement.clientHeight/2-150+(document.documentElement.scrollTop||document.body.scrollTop)];this._dialogInput.css("left",this._pos[0]+20+"px").css("top",this._pos[1]+"px");a.settings.onSelect=c;this._inDialog=true;this.dpDiv.addClass(this._dialogClass);this._showDatepicker(this._dialogInput[0]);d.blockUI&&d.blockUI(this.dpDiv);d.data(this._dialogInput[0],"datepicker",a);return this},_destroyDatepicker:function(a){var b= +d(a),c=d.data(a,"datepicker");if(b.hasClass(this.markerClassName)){var e=a.nodeName.toLowerCase();d.removeData(a,"datepicker");if(e=="input"){c.append.remove();c.trigger.remove();b.removeClass(this.markerClassName).unbind("focus",this._showDatepicker).unbind("keydown",this._doKeyDown).unbind("keypress",this._doKeyPress).unbind("keyup",this._doKeyUp)}else if(e=="div"||e=="span")b.removeClass(this.markerClassName).empty()}},_enableDatepicker:function(a){var b=d(a),c=d.data(a,"datepicker");if(b.hasClass(this.markerClassName)){var e= +a.nodeName.toLowerCase();if(e=="input"){a.disabled=false;c.trigger.filter("button").each(function(){this.disabled=false}).end().filter("img").css({opacity:"1.0",cursor:""})}else if(e=="div"||e=="span"){b=b.children("."+this._inlineClass);b.children().removeClass("ui-state-disabled");b.find("select.ui-datepicker-month, select.ui-datepicker-year").removeAttr("disabled")}this._disabledInputs=d.map(this._disabledInputs,function(f){return f==a?null:f})}},_disableDatepicker:function(a){var b=d(a),c=d.data(a, +"datepicker");if(b.hasClass(this.markerClassName)){var e=a.nodeName.toLowerCase();if(e=="input"){a.disabled=true;c.trigger.filter("button").each(function(){this.disabled=true}).end().filter("img").css({opacity:"0.5",cursor:"default"})}else if(e=="div"||e=="span"){b=b.children("."+this._inlineClass);b.children().addClass("ui-state-disabled");b.find("select.ui-datepicker-month, select.ui-datepicker-year").attr("disabled","disabled")}this._disabledInputs=d.map(this._disabledInputs,function(f){return f== +a?null:f});this._disabledInputs[this._disabledInputs.length]=a}},_isDisabledDatepicker:function(a){if(!a)return false;for(var b=0;b-1}},_doKeyUp:function(a){a=d.datepicker._getInst(a.target);if(a.input.val()!=a.lastVal)try{if(d.datepicker.parseDate(d.datepicker._get(a,"dateFormat"),a.input?a.input.val():null,d.datepicker._getFormatConfig(a))){d.datepicker._setDateFromField(a);d.datepicker._updateAlternate(a);d.datepicker._updateDatepicker(a)}}catch(b){d.datepicker.log(b)}return true},_showDatepicker:function(a){a=a.target||a;if(a.nodeName.toLowerCase()!="input")a=d("input", +a.parentNode)[0];if(!(d.datepicker._isDisabledDatepicker(a)||d.datepicker._lastInput==a)){var b=d.datepicker._getInst(a);if(d.datepicker._curInst&&d.datepicker._curInst!=b){d.datepicker._datepickerShowing&&d.datepicker._triggerOnClose(d.datepicker._curInst);d.datepicker._curInst.dpDiv.stop(true,true)}var c=d.datepicker._get(b,"beforeShow");c=c?c.apply(a,[a,b]):{};if(c!==false){H(b.settings,c);b.lastVal=null;d.datepicker._lastInput=a;d.datepicker._setDateFromField(b);if(d.datepicker._inDialog)a.value= +"";if(!d.datepicker._pos){d.datepicker._pos=d.datepicker._findPos(a);d.datepicker._pos[1]+=a.offsetHeight}var e=false;d(a).parents().each(function(){e|=d(this).css("position")=="fixed";return!e});if(e&&d.browser.opera){d.datepicker._pos[0]-=document.documentElement.scrollLeft;d.datepicker._pos[1]-=document.documentElement.scrollTop}c={left:d.datepicker._pos[0],top:d.datepicker._pos[1]};d.datepicker._pos=null;b.dpDiv.empty();b.dpDiv.css({position:"absolute",display:"block",top:"-1000px"});d.datepicker._updateDatepicker(b); +c=d.datepicker._checkOffset(b,c,e);b.dpDiv.css({position:d.datepicker._inDialog&&d.blockUI?"static":e?"fixed":"absolute",display:"none",left:c.left+"px",top:c.top+"px"});if(!b.inline){c=d.datepicker._get(b,"showAnim");var f=d.datepicker._get(b,"duration"),h=function(){var i=b.dpDiv.find("iframe.ui-datepicker-cover");if(i.length){var g=d.datepicker._getBorders(b.dpDiv);i.css({left:-g[0],top:-g[1],width:b.dpDiv.outerWidth(),height:b.dpDiv.outerHeight()})}};b.dpDiv.zIndex(d(a).zIndex()+1);d.datepicker._datepickerShowing= +true;d.effects&&d.effects[c]?b.dpDiv.show(c,d.datepicker._get(b,"showOptions"),f,h):b.dpDiv[c||"show"](c?f:null,h);if(!c||!f)h();b.input.is(":visible")&&!b.input.is(":disabled")&&b.input.focus();d.datepicker._curInst=b}}}},_updateDatepicker:function(a){this.maxRows=4;var b=d.datepicker._getBorders(a.dpDiv);J=a;a.dpDiv.empty().append(this._generateHTML(a));var c=a.dpDiv.find("iframe.ui-datepicker-cover");c.length&&c.css({left:-b[0],top:-b[1],width:a.dpDiv.outerWidth(),height:a.dpDiv.outerHeight()}); +a.dpDiv.find("."+this._dayOverClass+" a").mouseover();b=this._getNumberOfMonths(a);c=b[1];a.dpDiv.removeClass("ui-datepicker-multi-2 ui-datepicker-multi-3 ui-datepicker-multi-4").width("");c>1&&a.dpDiv.addClass("ui-datepicker-multi-"+c).css("width",17*c+"em");a.dpDiv[(b[0]!=1||b[1]!=1?"add":"remove")+"Class"]("ui-datepicker-multi");a.dpDiv[(this._get(a,"isRTL")?"add":"remove")+"Class"]("ui-datepicker-rtl");a==d.datepicker._curInst&&d.datepicker._datepickerShowing&&a.input&&a.input.is(":visible")&& +!a.input.is(":disabled")&&a.input[0]!=document.activeElement&&a.input.focus();if(a.yearshtml){var e=a.yearshtml;setTimeout(function(){e===a.yearshtml&&a.yearshtml&&a.dpDiv.find("select.ui-datepicker-year:first").replaceWith(a.yearshtml);e=a.yearshtml=null},0)}},_getBorders:function(a){var b=function(c){return{thin:1,medium:2,thick:3}[c]||c};return[parseFloat(b(a.css("border-left-width"))),parseFloat(b(a.css("border-top-width")))]},_checkOffset:function(a,b,c){var e=a.dpDiv.outerWidth(),f=a.dpDiv.outerHeight(), +h=a.input?a.input.outerWidth():0,i=a.input?a.input.outerHeight():0,g=document.documentElement.clientWidth+d(document).scrollLeft(),j=document.documentElement.clientHeight+d(document).scrollTop();b.left-=this._get(a,"isRTL")?e-h:0;b.left-=c&&b.left==a.input.offset().left?d(document).scrollLeft():0;b.top-=c&&b.top==a.input.offset().top+i?d(document).scrollTop():0;b.left-=Math.min(b.left,b.left+e>g&&g>e?Math.abs(b.left+e-g):0);b.top-=Math.min(b.top,b.top+f>j&&j>f?Math.abs(f+i):0);return b},_findPos:function(a){for(var b= +this._get(this._getInst(a),"isRTL");a&&(a.type=="hidden"||a.nodeType!=1||d.expr.filters.hidden(a));)a=a[b?"previousSibling":"nextSibling"];a=d(a).offset();return[a.left,a.top]},_triggerOnClose:function(a){var b=this._get(a,"onClose");if(b)b.apply(a.input?a.input[0]:null,[a.input?a.input.val():"",a])},_hideDatepicker:function(a){var b=this._curInst;if(!(!b||a&&b!=d.data(a,"datepicker")))if(this._datepickerShowing){a=this._get(b,"showAnim");var c=this._get(b,"duration"),e=function(){d.datepicker._tidyDialog(b); +this._curInst=null};d.effects&&d.effects[a]?b.dpDiv.hide(a,d.datepicker._get(b,"showOptions"),c,e):b.dpDiv[a=="slideDown"?"slideUp":a=="fadeIn"?"fadeOut":"hide"](a?c:null,e);a||e();d.datepicker._triggerOnClose(b);this._datepickerShowing=false;this._lastInput=null;if(this._inDialog){this._dialogInput.css({position:"absolute",left:"0",top:"-100px"});if(d.blockUI){d.unblockUI();d("body").append(this.dpDiv)}}this._inDialog=false}},_tidyDialog:function(a){a.dpDiv.removeClass(this._dialogClass).unbind(".ui-datepicker-calendar")}, +_checkExternalClick:function(a){if(d.datepicker._curInst){a=d(a.target);a[0].id!=d.datepicker._mainDivId&&a.parents("#"+d.datepicker._mainDivId).length==0&&!a.hasClass(d.datepicker.markerClassName)&&!a.hasClass(d.datepicker._triggerClass)&&d.datepicker._datepickerShowing&&!(d.datepicker._inDialog&&d.blockUI)&&d.datepicker._hideDatepicker()}},_adjustDate:function(a,b,c){a=d(a);var e=this._getInst(a[0]);if(!this._isDisabledDatepicker(a[0])){this._adjustInstDate(e,b+(c=="M"?this._get(e,"showCurrentAtPos"): +0),c);this._updateDatepicker(e)}},_gotoToday:function(a){a=d(a);var b=this._getInst(a[0]);if(this._get(b,"gotoCurrent")&&b.currentDay){b.selectedDay=b.currentDay;b.drawMonth=b.selectedMonth=b.currentMonth;b.drawYear=b.selectedYear=b.currentYear}else{var c=new Date;b.selectedDay=c.getDate();b.drawMonth=b.selectedMonth=c.getMonth();b.drawYear=b.selectedYear=c.getFullYear()}this._notifyChange(b);this._adjustDate(a)},_selectMonthYear:function(a,b,c){a=d(a);var e=this._getInst(a[0]);e["selected"+(c=="M"? +"Month":"Year")]=e["draw"+(c=="M"?"Month":"Year")]=parseInt(b.options[b.selectedIndex].value,10);this._notifyChange(e);this._adjustDate(a)},_selectDay:function(a,b,c,e){var f=d(a);if(!(d(e).hasClass(this._unselectableClass)||this._isDisabledDatepicker(f[0]))){f=this._getInst(f[0]);f.selectedDay=f.currentDay=d("a",e).html();f.selectedMonth=f.currentMonth=b;f.selectedYear=f.currentYear=c;this._selectDate(a,this._formatDate(f,f.currentDay,f.currentMonth,f.currentYear))}},_clearDate:function(a){a=d(a); +this._getInst(a[0]);this._selectDate(a,"")},_selectDate:function(a,b){a=this._getInst(d(a)[0]);b=b!=null?b:this._formatDate(a);a.input&&a.input.val(b);this._updateAlternate(a);var c=this._get(a,"onSelect");if(c)c.apply(a.input?a.input[0]:null,[b,a]);else a.input&&a.input.trigger("change");if(a.inline)this._updateDatepicker(a);else{this._hideDatepicker();this._lastInput=a.input[0];typeof a.input[0]!="object"&&a.input.focus();this._lastInput=null}},_updateAlternate:function(a){var b=this._get(a,"altField"); +if(b){var c=this._get(a,"altFormat")||this._get(a,"dateFormat"),e=this._getDate(a),f=this.formatDate(c,e,this._getFormatConfig(a));d(b).each(function(){d(this).val(f)})}},noWeekends:function(a){a=a.getDay();return[a>0&&a<6,""]},iso8601Week:function(a){a=new Date(a.getTime());a.setDate(a.getDate()+4-(a.getDay()||7));var b=a.getTime();a.setMonth(0);a.setDate(1);return Math.floor(Math.round((b-a)/864E5)/7)+1},parseDate:function(a,b,c){if(a==null||b==null)throw"Invalid arguments";b=typeof b=="object"? +b.toString():b+"";if(b=="")return null;var e=(c?c.shortYearCutoff:null)||this._defaults.shortYearCutoff;e=typeof e!="string"?e:(new Date).getFullYear()%100+parseInt(e,10);for(var f=(c?c.dayNamesShort:null)||this._defaults.dayNamesShort,h=(c?c.dayNames:null)||this._defaults.dayNames,i=(c?c.monthNamesShort:null)||this._defaults.monthNamesShort,g=(c?c.monthNames:null)||this._defaults.monthNames,j=c=-1,l=-1,u=-1,k=false,o=function(p){(p=A+1-1){j=1;l=u;do{e=this._getDaysInMonth(c,j-1);if(l<=e)break;j++;l-=e}while(1)}v=this._daylightSavingAdjust(new Date(c,j-1,l));if(v.getFullYear()!=c||v.getMonth()+1!=j||v.getDate()!=l)throw"Invalid date";return v},ATOM:"yy-mm-dd", +COOKIE:"D, dd M yy",ISO_8601:"yy-mm-dd",RFC_822:"D, d M y",RFC_850:"DD, dd-M-y",RFC_1036:"D, d M y",RFC_1123:"D, d M yy",RFC_2822:"D, d M yy",RSS:"D, d M y",TICKS:"!",TIMESTAMP:"@",W3C:"yy-mm-dd",_ticksTo1970:(718685+Math.floor(492.5)-Math.floor(19.7)+Math.floor(4.925))*24*60*60*1E7,formatDate:function(a,b,c){if(!b)return"";var e=(c?c.dayNamesShort:null)||this._defaults.dayNamesShort,f=(c?c.dayNames:null)||this._defaults.dayNames,h=(c?c.monthNamesShort:null)||this._defaults.monthNamesShort;c=(c?c.monthNames: +null)||this._defaults.monthNames;var i=function(o){(o=k+1 +12?a.getHours()+2:0);return a},_setDate:function(a,b,c){var e=!b,f=a.selectedMonth,h=a.selectedYear;b=this._restrictMinMax(a,this._determineDate(a,b,new Date));a.selectedDay=a.currentDay=b.getDate();a.drawMonth=a.selectedMonth=a.currentMonth=b.getMonth();a.drawYear=a.selectedYear=a.currentYear=b.getFullYear();if((f!=a.selectedMonth||h!=a.selectedYear)&&!c)this._notifyChange(a);this._adjustInstDate(a);if(a.input)a.input.val(e?"":this._formatDate(a))},_getDate:function(a){return!a.currentYear||a.input&& +a.input.val()==""?null:this._daylightSavingAdjust(new Date(a.currentYear,a.currentMonth,a.currentDay))},_generateHTML:function(a){var b=new Date;b=this._daylightSavingAdjust(new Date(b.getFullYear(),b.getMonth(),b.getDate()));var c=this._get(a,"isRTL"),e=this._get(a,"showButtonPanel"),f=this._get(a,"hideIfNoPrevNext"),h=this._get(a,"navigationAsDateFormat"),i=this._getNumberOfMonths(a),g=this._get(a,"showCurrentAtPos"),j=this._get(a,"stepMonths"),l=i[0]!=1||i[1]!=1,u=this._daylightSavingAdjust(!a.currentDay? +new Date(9999,9,9):new Date(a.currentYear,a.currentMonth,a.currentDay)),k=this._getMinMaxDate(a,"min"),o=this._getMinMaxDate(a,"max");g=a.drawMonth-g;var m=a.drawYear;if(g<0){g+=12;m--}if(o){var n=this._daylightSavingAdjust(new Date(o.getFullYear(),o.getMonth()-i[0]*i[1]+1,o.getDate()));for(n=k&&nn;){g--;if(g<0){g=11;m--}}}a.drawMonth=g;a.drawYear=m;n=this._get(a,"prevText");n=!h?n:this.formatDate(n,this._daylightSavingAdjust(new Date(m,g-j,1)),this._getFormatConfig(a)); +n=this._canAdjustMonth(a,-1,m,g)?''+n+"":f?"":''+n+"";var s=this._get(a,"nextText");s=!h?s:this.formatDate(s,this._daylightSavingAdjust(new Date(m, +g+j,1)),this._getFormatConfig(a));f=this._canAdjustMonth(a,+1,m,g)?''+s+"":f?"":''+s+"";j=this._get(a,"currentText");s=this._get(a,"gotoCurrent")&& +a.currentDay?u:b;j=!h?j:this.formatDate(j,s,this._getFormatConfig(a));h=!a.inline?'":"";e=e?'
'+(c?h:"")+(this._isInRange(a,s)?'":"")+(c?"":h)+"
":"";h=parseInt(this._get(a,"firstDay"),10);h=isNaN(h)?0:h;j=this._get(a,"showWeek");s=this._get(a,"dayNames");this._get(a,"dayNamesShort");var q=this._get(a,"dayNamesMin"),A=this._get(a,"monthNames"),v=this._get(a,"monthNamesShort"),p=this._get(a,"beforeShowDay"),D=this._get(a,"showOtherMonths"),K=this._get(a,"selectOtherMonths");this._get(a,"calculateWeek");for(var E=this._getDefaultDate(a),w="",x=0;x1)switch(G){case 0:y+=" ui-datepicker-group-first";t=" ui-corner-"+(c?"right":"left");break;case i[1]-1:y+=" ui-datepicker-group-last";t=" ui-corner-"+(c?"left":"right");break;default:y+=" ui-datepicker-group-middle";t="";break}y+='">'}y+='
'+(/all|left/.test(t)&& +x==0?c?f:n:"")+(/all|right/.test(t)&&x==0?c?n:f:"")+this._generateMonthYearHeader(a,g,m,k,o,x>0||G>0,A,v)+'
';var z=j?'":"";for(t=0;t<7;t++){var r=(t+h)%7;z+="=5?' class="ui-datepicker-week-end"':"")+'>'+q[r]+""}y+=z+"";z=this._getDaysInMonth(m,g);if(m==a.selectedYear&&g==a.selectedMonth)a.selectedDay=Math.min(a.selectedDay, +z);t=(this._getFirstDayOfMonth(m,g)-h+7)%7;z=Math.ceil((t+z)/7);this.maxRows=z=l?this.maxRows>z?this.maxRows:z:z;r=this._daylightSavingAdjust(new Date(m,g,1-t));for(var Q=0;Q";var R=!j?"":'";for(t=0;t<7;t++){var I=p?p.apply(a.input?a.input[0]:null,[r]):[true,""],F=r.getMonth()!=g,L=F&&!K||!I[0]||k&&ro;R+='";r.setDate(r.getDate()+1);r=this._daylightSavingAdjust(r)}y+=R+""}g++;if(g>11){g=0;m++}y+="
'+this._get(a,"weekHeader")+"
'+this._get(a,"calculateWeek")(r)+""+(F&&!D?" ":L?''+ +r.getDate()+"":''+r.getDate()+"")+"
"+(l?""+(i[0]>0&&G==i[1]-1?'
':""):"");O+=y}w+=O}w+=e+(d.browser.msie&&parseInt(d.browser.version,10)<7&&!a.inline?'': +"");a._keyEvent=false;return w},_generateMonthYearHeader:function(a,b,c,e,f,h,i,g){var j=this._get(a,"changeMonth"),l=this._get(a,"changeYear"),u=this._get(a,"showMonthAfterYear"),k='
',o="";if(h||!j)o+=''+i[b]+"";else{i=e&&e.getFullYear()==c;var m=f&&f.getFullYear()==c;o+='"}u||(k+=o+(h||!(j&&l)?" ":""));if(!a.yearshtml){a.yearshtml="";if(h||!l)k+=''+c+"";else{g=this._get(a,"yearRange").split(":");var s=(new Date).getFullYear();i=function(q){q=q.match(/c[+-].*/)?c+parseInt(q.substring(1),10):q.match(/[+-].*/)?s+parseInt(q,10):parseInt(q,10);return isNaN(q)?s:q};b=i(g[0]);g=Math.max(b,i(g[1]||""));b=e?Math.max(b, +e.getFullYear()):b;g=f?Math.min(g,f.getFullYear()):g;for(a.yearshtml+='";k+=a.yearshtml;a.yearshtml=null}}k+=this._get(a,"yearSuffix");if(u)k+=(h||!(j&&l)?" ":"")+o;k+="
";return k},_adjustInstDate:function(a,b,c){var e=a.drawYear+(c=="Y"?b:0),f=a.drawMonth+ +(c=="M"?b:0);b=Math.min(a.selectedDay,this._getDaysInMonth(e,f))+(c=="D"?b:0);e=this._restrictMinMax(a,this._daylightSavingAdjust(new Date(e,f,b)));a.selectedDay=e.getDate();a.drawMonth=a.selectedMonth=e.getMonth();a.drawYear=a.selectedYear=e.getFullYear();if(c=="M"||c=="Y")this._notifyChange(a)},_restrictMinMax:function(a,b){var c=this._getMinMaxDate(a,"min");a=this._getMinMaxDate(a,"max");b=c&&ba?a:b},_notifyChange:function(a){var b=this._get(a,"onChangeMonthYear");if(b)b.apply(a.input? +a.input[0]:null,[a.selectedYear,a.selectedMonth+1,a])},_getNumberOfMonths:function(a){a=this._get(a,"numberOfMonths");return a==null?[1,1]:typeof a=="number"?[1,a]:a},_getMinMaxDate:function(a,b){return this._determineDate(a,this._get(a,b+"Date"),null)},_getDaysInMonth:function(a,b){return 32-this._daylightSavingAdjust(new Date(a,b,32)).getDate()},_getFirstDayOfMonth:function(a,b){return(new Date(a,b,1)).getDay()},_canAdjustMonth:function(a,b,c,e){var f=this._getNumberOfMonths(a);c=this._daylightSavingAdjust(new Date(c, +e+(b<0?b:f[0]*f[1]),1));b<0&&c.setDate(this._getDaysInMonth(c.getFullYear(),c.getMonth()));return this._isInRange(a,c)},_isInRange:function(a,b){var c=this._getMinMaxDate(a,"min");a=this._getMinMaxDate(a,"max");return(!c||b.getTime()>=c.getTime())&&(!a||b.getTime()<=a.getTime())},_getFormatConfig:function(a){var b=this._get(a,"shortYearCutoff");b=typeof b!="string"?b:(new Date).getFullYear()%100+parseInt(b,10);return{shortYearCutoff:b,dayNamesShort:this._get(a,"dayNamesShort"),dayNames:this._get(a, +"dayNames"),monthNamesShort:this._get(a,"monthNamesShort"),monthNames:this._get(a,"monthNames")}},_formatDate:function(a,b,c,e){if(!b){a.currentDay=a.selectedDay;a.currentMonth=a.selectedMonth;a.currentYear=a.selectedYear}b=b?typeof b=="object"?b:this._daylightSavingAdjust(new Date(e,c,b)):this._daylightSavingAdjust(new Date(a.currentYear,a.currentMonth,a.currentDay));return this.formatDate(this._get(a,"dateFormat"),b,this._getFormatConfig(a))}});d.fn.datepicker=function(a){if(!this.length)return this; +if(!d.datepicker.initialized){d(document).mousedown(d.datepicker._checkExternalClick).find("body").append(d.datepicker.dpDiv);d.datepicker.initialized=true}var b=Array.prototype.slice.call(arguments,1);if(typeof a=="string"&&(a=="isDisabled"||a=="getDate"||a=="widget"))return d.datepicker["_"+a+"Datepicker"].apply(d.datepicker,[this[0]].concat(b));if(a=="option"&&arguments.length==2&&typeof arguments[1]=="string")return d.datepicker["_"+a+"Datepicker"].apply(d.datepicker,[this[0]].concat(b));return this.each(function(){typeof a== +"string"?d.datepicker["_"+a+"Datepicker"].apply(d.datepicker,[this].concat(b)):d.datepicker._attachDatepicker(this,a)})};d.datepicker=new M;d.datepicker.initialized=false;d.datepicker.uuid=(new Date).getTime();d.datepicker.version="1.8.16";window["DP_jQuery_"+B]=d})(jQuery); +;/* + * jQuery UI Effects 1.8.16 + * + * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Effects/ + */ +jQuery.effects||function(f,j){function m(c){var a;if(c&&c.constructor==Array&&c.length==3)return c;if(a=/rgb\(\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*\)/.exec(c))return[parseInt(a[1],10),parseInt(a[2],10),parseInt(a[3],10)];if(a=/rgb\(\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*\)/.exec(c))return[parseFloat(a[1])*2.55,parseFloat(a[2])*2.55,parseFloat(a[3])*2.55];if(a=/#([a-fA-F0-9]{2})([a-fA-F0-9]{2})([a-fA-F0-9]{2})/.exec(c))return[parseInt(a[1], +16),parseInt(a[2],16),parseInt(a[3],16)];if(a=/#([a-fA-F0-9])([a-fA-F0-9])([a-fA-F0-9])/.exec(c))return[parseInt(a[1]+a[1],16),parseInt(a[2]+a[2],16),parseInt(a[3]+a[3],16)];if(/rgba\(0, 0, 0, 0\)/.exec(c))return n.transparent;return n[f.trim(c).toLowerCase()]}function s(c,a){var b;do{b=f.curCSS(c,a);if(b!=""&&b!="transparent"||f.nodeName(c,"body"))break;a="backgroundColor"}while(c=c.parentNode);return m(b)}function o(){var c=document.defaultView?document.defaultView.getComputedStyle(this,null):this.currentStyle, +a={},b,d;if(c&&c.length&&c[0]&&c[c[0]])for(var e=c.length;e--;){b=c[e];if(typeof c[b]=="string"){d=b.replace(/\-(\w)/g,function(g,h){return h.toUpperCase()});a[d]=c[b]}}else for(b in c)if(typeof c[b]==="string")a[b]=c[b];return a}function p(c){var a,b;for(a in c){b=c[a];if(b==null||f.isFunction(b)||a in t||/scrollbar/.test(a)||!/color/i.test(a)&&isNaN(parseFloat(b)))delete c[a]}return c}function u(c,a){var b={_:0},d;for(d in a)if(c[d]!=a[d])b[d]=a[d];return b}function k(c,a,b,d){if(typeof c=="object"){d= +a;b=null;a=c;c=a.effect}if(f.isFunction(a)){d=a;b=null;a={}}if(typeof a=="number"||f.fx.speeds[a]){d=b;b=a;a={}}if(f.isFunction(b)){d=b;b=null}a=a||{};b=b||a.duration;b=f.fx.off?0:typeof b=="number"?b:b in f.fx.speeds?f.fx.speeds[b]:f.fx.speeds._default;d=d||a.complete;return[c,a,b,d]}function l(c){if(!c||typeof c==="number"||f.fx.speeds[c])return true;if(typeof c==="string"&&!f.effects[c])return true;return false}f.effects={};f.each(["backgroundColor","borderBottomColor","borderLeftColor","borderRightColor", +"borderTopColor","borderColor","color","outlineColor"],function(c,a){f.fx.step[a]=function(b){if(!b.colorInit){b.start=s(b.elem,a);b.end=m(b.end);b.colorInit=true}b.elem.style[a]="rgb("+Math.max(Math.min(parseInt(b.pos*(b.end[0]-b.start[0])+b.start[0],10),255),0)+","+Math.max(Math.min(parseInt(b.pos*(b.end[1]-b.start[1])+b.start[1],10),255),0)+","+Math.max(Math.min(parseInt(b.pos*(b.end[2]-b.start[2])+b.start[2],10),255),0)+")"}});var n={aqua:[0,255,255],azure:[240,255,255],beige:[245,245,220],black:[0, +0,0],blue:[0,0,255],brown:[165,42,42],cyan:[0,255,255],darkblue:[0,0,139],darkcyan:[0,139,139],darkgrey:[169,169,169],darkgreen:[0,100,0],darkkhaki:[189,183,107],darkmagenta:[139,0,139],darkolivegreen:[85,107,47],darkorange:[255,140,0],darkorchid:[153,50,204],darkred:[139,0,0],darksalmon:[233,150,122],darkviolet:[148,0,211],fuchsia:[255,0,255],gold:[255,215,0],green:[0,128,0],indigo:[75,0,130],khaki:[240,230,140],lightblue:[173,216,230],lightcyan:[224,255,255],lightgreen:[144,238,144],lightgrey:[211, +211,211],lightpink:[255,182,193],lightyellow:[255,255,224],lime:[0,255,0],magenta:[255,0,255],maroon:[128,0,0],navy:[0,0,128],olive:[128,128,0],orange:[255,165,0],pink:[255,192,203],purple:[128,0,128],violet:[128,0,128],red:[255,0,0],silver:[192,192,192],white:[255,255,255],yellow:[255,255,0],transparent:[255,255,255]},q=["add","remove","toggle"],t={border:1,borderBottom:1,borderColor:1,borderLeft:1,borderRight:1,borderTop:1,borderWidth:1,margin:1,padding:1};f.effects.animateClass=function(c,a,b, +d){if(f.isFunction(b)){d=b;b=null}return this.queue(function(){var e=f(this),g=e.attr("style")||" ",h=p(o.call(this)),r,v=e.attr("class");f.each(q,function(w,i){c[i]&&e[i+"Class"](c[i])});r=p(o.call(this));e.attr("class",v);e.animate(u(h,r),{queue:false,duration:a,easing:b,complete:function(){f.each(q,function(w,i){c[i]&&e[i+"Class"](c[i])});if(typeof e.attr("style")=="object"){e.attr("style").cssText="";e.attr("style").cssText=g}else e.attr("style",g);d&&d.apply(this,arguments);f.dequeue(this)}})})}; +f.fn.extend({_addClass:f.fn.addClass,addClass:function(c,a,b,d){return a?f.effects.animateClass.apply(this,[{add:c},a,b,d]):this._addClass(c)},_removeClass:f.fn.removeClass,removeClass:function(c,a,b,d){return a?f.effects.animateClass.apply(this,[{remove:c},a,b,d]):this._removeClass(c)},_toggleClass:f.fn.toggleClass,toggleClass:function(c,a,b,d,e){return typeof a=="boolean"||a===j?b?f.effects.animateClass.apply(this,[a?{add:c}:{remove:c},b,d,e]):this._toggleClass(c,a):f.effects.animateClass.apply(this, +[{toggle:c},a,b,d])},switchClass:function(c,a,b,d,e){return f.effects.animateClass.apply(this,[{add:a,remove:c},b,d,e])}});f.extend(f.effects,{version:"1.8.16",save:function(c,a){for(var b=0;b").addClass("ui-effects-wrapper").css({fontSize:"100%",background:"transparent",border:"none",margin:0,padding:0}), +d=document.activeElement;c.wrap(b);if(c[0]===d||f.contains(c[0],d))f(d).focus();b=c.parent();if(c.css("position")=="static"){b.css({position:"relative"});c.css({position:"relative"})}else{f.extend(a,{position:c.css("position"),zIndex:c.css("z-index")});f.each(["top","left","bottom","right"],function(e,g){a[g]=c.css(g);if(isNaN(parseInt(a[g],10)))a[g]="auto"});c.css({position:"relative",top:0,left:0,right:"auto",bottom:"auto"})}return b.css(a).show()},removeWrapper:function(c){var a,b=document.activeElement; +if(c.parent().is(".ui-effects-wrapper")){a=c.parent().replaceWith(c);if(c[0]===b||f.contains(c[0],b))f(b).focus();return a}return c},setTransition:function(c,a,b,d){d=d||{};f.each(a,function(e,g){unit=c.cssUnit(g);if(unit[0]>0)d[g]=unit[0]*b+unit[1]});return d}});f.fn.extend({effect:function(c){var a=k.apply(this,arguments),b={options:a[1],duration:a[2],callback:a[3]};a=b.options.mode;var d=f.effects[c];if(f.fx.off||!d)return a?this[a](b.duration,b.callback):this.each(function(){b.callback&&b.callback.call(this)}); +return d.call(this,b)},_show:f.fn.show,show:function(c){if(l(c))return this._show.apply(this,arguments);else{var a=k.apply(this,arguments);a[1].mode="show";return this.effect.apply(this,a)}},_hide:f.fn.hide,hide:function(c){if(l(c))return this._hide.apply(this,arguments);else{var a=k.apply(this,arguments);a[1].mode="hide";return this.effect.apply(this,a)}},__toggle:f.fn.toggle,toggle:function(c){if(l(c)||typeof c==="boolean"||f.isFunction(c))return this.__toggle.apply(this,arguments);else{var a=k.apply(this, +arguments);a[1].mode="toggle";return this.effect.apply(this,a)}},cssUnit:function(c){var a=this.css(c),b=[];f.each(["em","px","%","pt"],function(d,e){if(a.indexOf(e)>0)b=[parseFloat(a),e]});return b}});f.easing.jswing=f.easing.swing;f.extend(f.easing,{def:"easeOutQuad",swing:function(c,a,b,d,e){return f.easing[f.easing.def](c,a,b,d,e)},easeInQuad:function(c,a,b,d,e){return d*(a/=e)*a+b},easeOutQuad:function(c,a,b,d,e){return-d*(a/=e)*(a-2)+b},easeInOutQuad:function(c,a,b,d,e){if((a/=e/2)<1)return d/ +2*a*a+b;return-d/2*(--a*(a-2)-1)+b},easeInCubic:function(c,a,b,d,e){return d*(a/=e)*a*a+b},easeOutCubic:function(c,a,b,d,e){return d*((a=a/e-1)*a*a+1)+b},easeInOutCubic:function(c,a,b,d,e){if((a/=e/2)<1)return d/2*a*a*a+b;return d/2*((a-=2)*a*a+2)+b},easeInQuart:function(c,a,b,d,e){return d*(a/=e)*a*a*a+b},easeOutQuart:function(c,a,b,d,e){return-d*((a=a/e-1)*a*a*a-1)+b},easeInOutQuart:function(c,a,b,d,e){if((a/=e/2)<1)return d/2*a*a*a*a+b;return-d/2*((a-=2)*a*a*a-2)+b},easeInQuint:function(c,a,b, +d,e){return d*(a/=e)*a*a*a*a+b},easeOutQuint:function(c,a,b,d,e){return d*((a=a/e-1)*a*a*a*a+1)+b},easeInOutQuint:function(c,a,b,d,e){if((a/=e/2)<1)return d/2*a*a*a*a*a+b;return d/2*((a-=2)*a*a*a*a+2)+b},easeInSine:function(c,a,b,d,e){return-d*Math.cos(a/e*(Math.PI/2))+d+b},easeOutSine:function(c,a,b,d,e){return d*Math.sin(a/e*(Math.PI/2))+b},easeInOutSine:function(c,a,b,d,e){return-d/2*(Math.cos(Math.PI*a/e)-1)+b},easeInExpo:function(c,a,b,d,e){return a==0?b:d*Math.pow(2,10*(a/e-1))+b},easeOutExpo:function(c, +a,b,d,e){return a==e?b+d:d*(-Math.pow(2,-10*a/e)+1)+b},easeInOutExpo:function(c,a,b,d,e){if(a==0)return b;if(a==e)return b+d;if((a/=e/2)<1)return d/2*Math.pow(2,10*(a-1))+b;return d/2*(-Math.pow(2,-10*--a)+2)+b},easeInCirc:function(c,a,b,d,e){return-d*(Math.sqrt(1-(a/=e)*a)-1)+b},easeOutCirc:function(c,a,b,d,e){return d*Math.sqrt(1-(a=a/e-1)*a)+b},easeInOutCirc:function(c,a,b,d,e){if((a/=e/2)<1)return-d/2*(Math.sqrt(1-a*a)-1)+b;return d/2*(Math.sqrt(1-(a-=2)*a)+1)+b},easeInElastic:function(c,a,b, +d,e){c=1.70158;var g=0,h=d;if(a==0)return b;if((a/=e)==1)return b+d;g||(g=e*0.3);if(h").css({position:"absolute",visibility:"visible",left:-f*(h/d),top:-e*(i/c)}).parent().addClass("ui-effects-explode").css({position:"absolute",overflow:"hidden",width:h/d,height:i/c,left:g.left+f*(h/d)+(a.options.mode=="show"?(f-Math.floor(d/2))*(h/d):0),top:g.top+e*(i/c)+(a.options.mode=="show"?(e-Math.floor(c/2))*(i/c):0),opacity:a.options.mode=="show"?0:1}).animate({left:g.left+f*(h/d)+(a.options.mode=="show"?0:(f-Math.floor(d/2))*(h/d)),top:g.top+ +e*(i/c)+(a.options.mode=="show"?0:(e-Math.floor(c/2))*(i/c)),opacity:a.options.mode=="show"?1:0},a.duration||500);setTimeout(function(){a.options.mode=="show"?b.css({visibility:"visible"}):b.css({visibility:"visible"}).hide();a.callback&&a.callback.apply(b[0]);b.dequeue();j("div.ui-effects-explode").remove()},a.duration||500)})}})(jQuery); +;/* + * jQuery UI Effects Fade 1.8.16 + * + * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Effects/Fade + * + * Depends: + * jquery.effects.core.js + */ +(function(b){b.effects.fade=function(a){return this.queue(function(){var c=b(this),d=b.effects.setMode(c,a.options.mode||"hide");c.animate({opacity:d},{queue:false,duration:a.duration,easing:a.options.easing,complete:function(){a.callback&&a.callback.apply(this,arguments);c.dequeue()}})})}})(jQuery); +;/* + * jQuery UI Effects Fold 1.8.16 + * + * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Effects/Fold + * + * Depends: + * jquery.effects.core.js + */ +(function(c){c.effects.fold=function(a){return this.queue(function(){var b=c(this),j=["position","top","bottom","left","right"],d=c.effects.setMode(b,a.options.mode||"hide"),g=a.options.size||15,h=!!a.options.horizFirst,k=a.duration?a.duration/2:c.fx.speeds._default/2;c.effects.save(b,j);b.show();var e=c.effects.createWrapper(b).css({overflow:"hidden"}),f=d=="show"!=h,l=f?["width","height"]:["height","width"];f=f?[e.width(),e.height()]:[e.height(),e.width()];var i=/([0-9]+)%/.exec(g);if(i)g=parseInt(i[1], +10)/100*f[d=="hide"?0:1];if(d=="show")e.css(h?{height:0,width:g}:{height:g,width:0});h={};i={};h[l[0]]=d=="show"?f[0]:g;i[l[1]]=d=="show"?f[1]:0;e.animate(h,k,a.options.easing).animate(i,k,a.options.easing,function(){d=="hide"&&b.hide();c.effects.restore(b,j);c.effects.removeWrapper(b);a.callback&&a.callback.apply(b[0],arguments);b.dequeue()})})}})(jQuery); +;/* + * jQuery UI Effects Highlight 1.8.16 + * + * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Effects/Highlight + * + * Depends: + * jquery.effects.core.js + */ +(function(b){b.effects.highlight=function(c){return this.queue(function(){var a=b(this),e=["backgroundImage","backgroundColor","opacity"],d=b.effects.setMode(a,c.options.mode||"show"),f={backgroundColor:a.css("backgroundColor")};if(d=="hide")f.opacity=0;b.effects.save(a,e);a.show().css({backgroundImage:"none",backgroundColor:c.options.color||"#ffff99"}).animate(f,{queue:false,duration:c.duration,easing:c.options.easing,complete:function(){d=="hide"&&a.hide();b.effects.restore(a,e);d=="show"&&!b.support.opacity&& +this.style.removeAttribute("filter");c.callback&&c.callback.apply(this,arguments);a.dequeue()}})})}})(jQuery); +;/* + * jQuery UI Effects Pulsate 1.8.16 + * + * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * http://docs.jquery.com/UI/Effects/Pulsate + * + * Depends: + * jquery.effects.core.js + */ +(function(d){d.effects.pulsate=function(a){return this.queue(function(){var b=d(this),c=d.effects.setMode(b,a.options.mode||"show");times=(a.options.times||5)*2-1;duration=a.duration?a.duration/2:d.fx.speeds._default/2;isVisible=b.is(":visible");animateTo=0;if(!isVisible){b.css("opacity",0).show();animateTo=1}if(c=="hide"&&isVisible||c=="show"&&!isVisible)times--;for(c=0;c').appendTo(document.body).addClass(a.options.className).css({top:d.top,left:d.left,height:b.innerHeight(),width:b.innerWidth(),position:"absolute"}).animate(c,a.duration,a.options.easing,function(){f.remove();a.callback&&a.callback.apply(b[0],arguments); +b.dequeue()})})}})(jQuery); +; \ No newline at end of file diff --git a/ckan/public/base/datapreview/vendor/slickgrid/2.0.1/jquery.event.drag-2.0.js b/ckan/public/base/datapreview/vendor/slickgrid/2.0.1/jquery.event.drag-2.0.js new file mode 100644 index 00000000000..2cb7fee0e2e --- /dev/null +++ b/ckan/public/base/datapreview/vendor/slickgrid/2.0.1/jquery.event.drag-2.0.js @@ -0,0 +1,6 @@ +/*! + * jquery.event.drag - v 2.0.0 + * Copyright (c) 2010 Three Dub Media - http://threedubmedia.com + * Open Source MIT License - http://threedubmedia.com/code/license + */ +;(function(f){f.fn.drag=function(b,a,d){var e=typeof b=="string"?b:"",k=f.isFunction(b)?b:f.isFunction(a)?a:null;if(e.indexOf("drag")!==0)e="drag"+e;d=(b==k?a:d)||{};return k?this.bind(e,d,k):this.trigger(e)};var i=f.event,h=i.special,c=h.drag={defaults:{which:1,distance:0,not:":input",handle:null,relative:false,drop:true,click:false},datakey:"dragdata",livekey:"livedrag",add:function(b){var a=f.data(this,c.datakey),d=b.data||{};a.related+=1;if(!a.live&&b.selector){a.live=true;i.add(this,"draginit."+ c.livekey,c.delegate)}f.each(c.defaults,function(e){if(d[e]!==undefined)a[e]=d[e]})},remove:function(){f.data(this,c.datakey).related-=1},setup:function(){if(!f.data(this,c.datakey)){var b=f.extend({related:0},c.defaults);f.data(this,c.datakey,b);i.add(this,"mousedown",c.init,b);this.attachEvent&&this.attachEvent("ondragstart",c.dontstart)}},teardown:function(){if(!f.data(this,c.datakey).related){f.removeData(this,c.datakey);i.remove(this,"mousedown",c.init);i.remove(this,"draginit",c.delegate);c.textselect(true); this.detachEvent&&this.detachEvent("ondragstart",c.dontstart)}},init:function(b){var a=b.data,d;if(!(a.which>0&&b.which!=a.which))if(!f(b.target).is(a.not))if(!(a.handle&&!f(b.target).closest(a.handle,b.currentTarget).length)){a.propagates=1;a.interactions=[c.interaction(this,a)];a.target=b.target;a.pageX=b.pageX;a.pageY=b.pageY;a.dragging=null;d=c.hijack(b,"draginit",a);if(a.propagates){if((d=c.flatten(d))&&d.length){a.interactions=[];f.each(d,function(){a.interactions.push(c.interaction(this,a))})}a.propagates= a.interactions.length;a.drop!==false&&h.drop&&h.drop.handler(b,a);c.textselect(false);i.add(document,"mousemove mouseup",c.handler,a);return false}}},interaction:function(b,a){return{drag:b,callback:new c.callback,droppable:[],offset:f(b)[a.relative?"position":"offset"]()||{top:0,left:0}}},handler:function(b){var a=b.data;switch(b.type){case !a.dragging&&"mousemove":if(Math.pow(b.pageX-a.pageX,2)+Math.pow(b.pageY-a.pageY,2)=0;f--)d[f]===e&&d.splice(f,1)};this.notify=function(e,f,g){for(var f=f||new G,g=g||this,A,ma=0;ma=this.fromRow&&d<=this.toRow&&e>=this.fromCell&&e<=this.toCell};this.toString=function(){return this.isSingleCell()?"("+this.fromRow+":"+this.fromCell+")":"("+this.fromRow+":"+this.fromCell+" - "+this.toRow+":"+this.toCell+")"}},NonDataRow:A,Group:ba,GroupTotals:g,EditorLock:f,GlobalEditorLock:new f}});ba.prototype= +new A;ba.prototype.equals=function(d){return this.value===d.value&&this.count===d.count&&this.collapsed===d.collapsed};g.prototype=new A})(jQuery);/* + + (c) 2009-2012 Michael Leibman + michael{dot}leibman{at}gmail{dot}com + http://github.com/mleibman/slickgrid + + Distributed under MIT license. + All rights reserved. + + SlickGrid v2.0 + + NOTES: + Cell/row DOM manipulations are done directly bypassing jQuery's DOM manipulation methods. + This increases the speed dramatically, but can only be done safely because there are no event handlers + or data associated with any cell/row DOM nodes. Cell editors must make sure they implement .destroy() + and do proper cleanup. +*/ +if("undefined"===typeof jQuery)throw"SlickGrid requires jquery module to be loaded";if(!jQuery.fn.drag)throw"SlickGrid requires jquery.event.drag module to be loaded";if("undefined"===typeof Slick)throw"slick.core.js not loaded"; +(function(e){e.extend(!0,window,{Slick:{Grid:function(ba,g,f,d){function fb(){if(!H){H=true;U=parseFloat(e.css(n[0],"width",true));Vb();ma(t);d.enableTextSelectionOnCells||v.bind("selectstart.ui",function(a){return e(a.target).is("input,textarea")});hb();Wb();ib();K();Xb();n.bind("resize.slickgrid",K);v.bind("scroll.slickgrid",jb);ca.bind("contextmenu.slickgrid",Yb).bind("click.slickgrid",Zb);Ka.bind("keydown.slickgrid",kb);B.bind("keydown.slickgrid",kb).bind("click.slickgrid",$b).bind("dblclick.slickgrid", +ac).bind("contextmenu.slickgrid",bc).bind("draginit",cc).bind("dragstart",dc).bind("drag",ec).bind("dragend",fc).delegate(".slick-cell","mouseenter",gc).delegate(".slick-cell","mouseleave",hc)}}function gb(a){for(var b=O.length;b>=0;b--)if(O[b]===a){O[b].destroy&&O[b].destroy();O.splice(b,1);break}}function Ub(){var a=e("
").appendTo("body"),b={width:a.width()-a[0].clientWidth,height:a.height()- +a[0].clientHeight};a.remove();return b}function Ja(a){for(var b=P,c=na?U-G.width:U,j=0,e=f.length;e--;)j=j+(f[e].width||La.width);P=d.fullWidthRows?Math.max(j,c):j;if(P!=b){B.width(P);da.width(P);lb=P>U-G.width}(P!=b||a)&&Ma()}function ma(a){a&&a.jquery&&a.attr("unselectable","on").css("MozUserSelect","none").bind("selectstart.ui",function(){return false})}function ic(){for(var a=1E6,b=e.browser.mozilla?5E6:1E9,c=e("
").appendTo(document.body);a<=b;){c.css("height",a+1E6); +if(c.height()!==a+1E6)break;else a=a+1E6}c.remove();return a}function Xb(){for(var a=B[0];(a=a.parentNode)!=document.body&&a!=null;)(a==v[0]||a.scrollWidth!=a.clientWidth||a.scrollHeight!=a.clientHeight)&&e(a).bind("scroll.slickgrid",mb)}function hb(){function a(){e(this).addClass("ui-state-hover")}function b(){e(this).removeClass("ui-state-hover")}t.empty();da.empty();I={};for(var c=0;c").html(""+j.name+"").width(j.width-V).attr("title",j.toolTip||j.name||"").data("fieldId",j.id).addClass(j.headerCssClass||"").appendTo(t);(d.enableColumnReorder||j.sortable)&&oa.hover(a,b);j.sortable&&oa.append("");d.showHeaderRow&&e("
").appendTo(da)}d.showHeaderRow&&e("
").appendTo(Q); +ta(r);nb();d.enableColumnReorder&&jc()}function Wb(){t.click(function(a){a.metaKey=a.metaKey||a.ctrlKey;if(!e(a.target).hasClass("slick-resizable-handle")){var b=e(a.target).closest(".slick-header-column");if(b.length){b=f[Na(b.data("fieldId"))];if(b.sortable&&p().commitCurrentEdit()){for(var c=null,j=0;j=g)){e(m);e("
").appendTo(m).bind("dragstart",function(g){if(!p().commitCurrentEdit())return false;c=g.pageX;e(this).parent().addClass("slick-header-column-active");var h=g=null;j.each(function(a,b){f[a].previousWidth=e(b).outerWidth()});if(d.forceFitColumns){h=g=0;for(a=k+1;a=0;a--){b=f[a];if(b.resizable){e=Math.max(b.minWidth||0,R);if(h&&b.previousWidth+h=0;a--){b=f[a];if(b.resizable)if(h&&b.maxWidth&&b.maxWidth-b.previousWidth
").appendTo(t);V=Pa=0;e.each(b,function(b,c){V=V+(parseFloat(a.css(c))||0)});e.each(c,function(b,c){Pa=Pa+(parseFloat(a.css(c))||0)});a.remove();var j=e("
").appendTo(B);a=e("").appendTo(j); +ua=va=0;e.each(b,function(b,c){ua=ua+(parseFloat(a.css(c))||0)});e.each(c,function(b,c){va=va+(parseFloat(a.css(c))||0)});j.remove();R=Math.max(V,ua)}function ib(){X=e("