diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000000..baf0efddc81 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,10 @@ +### CKAN Version if known (or site URL) + + +### Please describe the expected behaviour + + +### Please describe the actual behaviour + + +### What steps can be taken to reproduce the issue? diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000000..e05120c4edd --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,15 @@ +Fixes # + +### Proposed fixes: + + + +### Features: + +- [ ] includes tests covering changes +- [ ] includes updated documentation +- [ ] includes user-visible changes +- [ ] includes API changes +- [ ] includes bugfix for possible backport + +Please [X] all the boxes above that apply diff --git a/.travis.yml b/.travis.yml index 7b04aabff2e..1815e0e5b33 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,20 @@ language: python +sudo: required python: - "2.6" - "2.7" env: - - PGVERSION=9.1 - - PGVERSION=8.4 + global: + - CKAN_POSTGRES_DB=ckan_test + - CKAN_POSTGRES_USER=ckan_default + - CKAN_POSTGRES_PWD=pass + - CKAN_DATASTORE_POSTGRES_DB=datastore_test + - CKAN_DATASTORE_POSTGRES_WRITE_USER=ckan_default + - CKAN_DATASTORE_POSTGRES_READ_USER=datastore_default + - CKAN_DATASTORE_POSTGRES_READ_PWD=pass + matrix: + - PGVERSION=9.1 + - PGVERSION=8.4 matrix: exclude: - python: "2.7" diff --git a/bin/postgres_init/1_create_ckan_db.sh b/bin/postgres_init/1_create_ckan_db.sh new file mode 100755 index 00000000000..bec5fe26a45 --- /dev/null +++ b/bin/postgres_init/1_create_ckan_db.sh @@ -0,0 +1,6 @@ +#!/bin/sh +psql -U postgres -c "CREATE USER ${CKAN_POSTGRES_USER} \ + WITH PASSWORD '${CKAN_POSTGRES_PWD}' \ + NOSUPERUSER NOCREATEDB NOCREATEROLE;" + +createdb -U postgres -e -O ${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB} -E utf-8 diff --git a/bin/postgres_init/2_create_ckan_datastore_db.sh b/bin/postgres_init/2_create_ckan_datastore_db.sh new file mode 100755 index 00000000000..c775b5749a9 --- /dev/null +++ b/bin/postgres_init/2_create_ckan_datastore_db.sh @@ -0,0 +1,14 @@ +#!/bin/sh +if [ "${CKAN_DATASTORE_POSTGRES_DB}" ]; then + psql -U postgres -c "CREATE USER ${CKAN_DATASTORE_POSTGRES_READ_USER} \ + WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_READ_PWD}' \ + NOSUPERUSER NOCREATEDB NOCREATEROLE;" + + if [ "${CKAN_DATASTORE_POSTGRES_WRITE_USER}" ] && [ "${CKAN_DATASTORE_POSTGRES_WRITE_PWD}" ]; then + psql -U postgres -c "CREATE USER ${CKAN_DATASTORE_POSTGRES_WRITE_USER} \ + WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_WRITE_PWD}' \ + NOSUPERUSER NOCREATEDB NOCREATEROLE;" + fi + + createdb -U postgres -e -O ${CKAN_DATASTORE_POSTGRES_WRITE_USER} ${CKAN_DATASTORE_POSTGRES_DB} -E utf-8 +fi diff --git a/bin/travis-install-dependencies b/bin/travis-install-dependencies index d9dacfbcdc7..8b9b1430d8f 100755 --- a/bin/travis-install-dependencies +++ b/bin/travis-install-dependencies @@ -25,10 +25,8 @@ fi sudo service postgresql restart # Setup postgres' users and databases -sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';" -sudo -u postgres psql -c "CREATE USER datastore_default WITH PASSWORD 'pass';" -sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;' -sudo -u postgres psql -c 'CREATE DATABASE datastore_test WITH OWNER ckan_default;' +sudo -E -u postgres ./bin/postgres_init/1_create_ckan_db.sh +sudo -E -u postgres ./bin/postgres_init/2_create_ckan_datastore_db.sh export PIP_USE_MIRRORS=true pip install -r requirements.txt --allow-all-external diff --git a/circle.yml b/circle.yml index 37215b0b47e..003e84a7a34 100644 --- a/circle.yml +++ b/circle.yml @@ -2,6 +2,13 @@ machine: environment: PIP_USE_MIRRORS: true + CKAN_POSTGRES_DB: ckan_test + CKAN_POSTGRES_USER: ckan_default + CKAN_POSTGRES_PWD: pass + CKAN_DATASTORE_POSTGRES_DB: datastore_test + CKAN_DATASTORE_POSTGRES_WRITE_USER: ckan_default + CKAN_DATASTORE_POSTGRES_READ_USER: datastore_default + CKAN_DATASTORE_POSTGRES_READ_PWD: pass dependencies: override: @@ -14,10 +21,8 @@ dependencies: database: post: - - sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';" - - sudo -u postgres psql -c "CREATE USER datastore_default WITH PASSWORD 'pass';" - - sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;' - - sudo -u postgres psql -c 'CREATE DATABASE datastore_test WITH OWNER ckan_default;' + - sudo -E -u postgres ./bin/postgres_init/1_create_ckan_db.sh + - sudo -E -u postgres ./bin/postgres_init/2_create_ckan_datastore_db.sh - sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default:pass@\/datastore_test/' test-core.ini - paster datastore -c test-core.ini set-permissions | sudo -u postgres psql diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index 299c424a51c..80cfb5a1df4 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -323,7 +323,10 @@ def _save_edit(self, id, context): context['message'] = data_dict.get('log_message', '') data_dict['id'] = id - if data_dict['password1'] and data_dict['password2']: + email_changed = data_dict['email'] != c.userobj.email + + if (data_dict['password1'] and data_dict['password2']) \ + or email_changed: identity = {'login': c.user, 'password': data_dict['old_password']} auth = authenticator.UsernamePasswordAuthenticator() diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 94f62c00429..e295a0cb2dc 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -186,9 +186,6 @@ class ManageDb(CkanCommand): db upgrade [version no.] - Data migrate db version - returns current version of data schema db dump FILE_PATH - dump to a pg_dump file - db simple-dump-csv FILE_PATH - dump just datasets in CSV format - db simple-dump-json FILE_PATH - dump just datasets in JSON format - db user-dump-csv FILE_PATH - dump user information to a CSV file db load FILE_PATH - load a pg_dump from a file db load-only FILE_PATH - load a pg_dump from a file but don\'t do the schema upgrade or search indexing @@ -239,12 +236,6 @@ def command(self): self.load() elif cmd == 'load-only': self.load(only_load=True) - elif cmd == 'simple-dump-csv': - self.simple_dump_csv() - elif cmd == 'simple-dump-json': - self.simple_dump_json() - elif cmd == 'user-dump-csv': - self.user_dump_csv() elif cmd == 'create-from-model': model.repo.create_db() if self.verbose: @@ -326,35 +317,6 @@ def load(self, only_load=False): print 'Now remember you have to call \'db upgrade\' and then \'search-index rebuild\'.' print 'Done' - def simple_dump_csv(self): - import ckan.model as model - if len(self.args) < 2: - print 'Need csv file path' - return - dump_filepath = self.args[1] - import ckan.lib.dumper as dumper - dump_file = open(dump_filepath, 'w') - dumper.SimpleDumper().dump(dump_file, format='csv') - - def simple_dump_json(self): - import ckan.model as model - if len(self.args) < 2: - print 'Need json file path' - return - dump_filepath = self.args[1] - import ckan.lib.dumper as dumper - dump_file = open(dump_filepath, 'w') - dumper.SimpleDumper().dump(dump_file, format='json') - - def user_dump_csv(self): - if len(self.args) < 2: - print 'Need csv file path' - return - dump_filepath = self.args[1] - import ckan.lib.dumper as dumper - dump_file = open(dump_filepath, 'w') - dumper.UserDumper().dump(dump_file) - def migrate_filestore(self): from ckan.model import Session import requests diff --git a/ckan/lib/dumper.py b/ckan/lib/dumper.py deleted file mode 100644 index 83431ca53b2..00000000000 --- a/ckan/lib/dumper.py +++ /dev/null @@ -1,108 +0,0 @@ -import csv - -import ckan.model as model -from ckan.common import json, OrderedDict - - -class SimpleDumper(object): - '''Dumps just package data but including tags, groups, license text etc''' - def dump(self, dump_file_obj, format='json', query=None): - if query is None: - query = model.Session.query(model.Package) - active = model.State.ACTIVE - query = query.filter_by(state=active) - if format == 'csv': - self.dump_csv(dump_file_obj, query) - elif format == 'json': - self.dump_json(dump_file_obj, query) - else: - raise Exception('Unknown format: %s' % format) - - def dump_csv(self, dump_file_obj, query): - row_dicts = [] - for pkg in query: - pkg_dict = pkg.as_dict() - # flatten dict - for name, value in pkg_dict.items()[:]: - if isinstance(value, (list, tuple)): - if value and isinstance(value[0], dict) and \ - name == 'resources': - for i, res in enumerate(value): - prefix = 'resource-%i' % i - pkg_dict[prefix + '-url'] = res['url'] - pkg_dict[prefix + '-format'] = res['format'] - pkg_dict[prefix + '-description'] = \ - res['description'] - else: - pkg_dict[name] = ' '.join(value) - if isinstance(value, dict): - for name_, value_ in value.items(): - pkg_dict[name_] = value_ - del pkg_dict[name] - row_dicts.append(pkg_dict) - writer = CsvWriter(row_dicts) - writer.save(dump_file_obj) - - def dump_json(self, dump_file_obj, query): - pkgs = [] - for pkg in query: - pkg_dict = pkg.as_dict() - pkgs.append(pkg_dict) - json.dump(pkgs, dump_file_obj, indent=4) - - -class CsvWriter: - def __init__(self, package_dict_list=None): - self._rows = [] - self._col_titles = [] - for row_dict in package_dict_list: - for key in row_dict.keys(): - if key not in self._col_titles: - self._col_titles.append(key) - for row_dict in package_dict_list: - self._add_row_dict(row_dict) - - def _add_row_dict(self, row_dict): - row = [] - for title in self._col_titles: - if title in row_dict: - if isinstance(row_dict[title], int): - row.append(row_dict[title]) - elif isinstance(row_dict[title], unicode): - row.append(row_dict[title].encode('utf8')) - else: - row.append(row_dict[title]) - else: - row.append(None) - self._rows.append(row) - - def save(self, file_obj): - writer = csv.writer(file_obj, quotechar='"', - quoting=csv.QUOTE_NONNUMERIC) - writer.writerow(self._col_titles) - for row in self._rows: - writer.writerow(row) - - -class UserDumper(object): - def dump(self, dump_file_obj): - query = model.Session.query(model.User) - query = query.order_by(model.User.created.asc()) - - columns = (('id', 'name', 'openid', 'fullname', 'email', 'created', - 'about')) - row_dicts = [] - for user in query: - row = OrderedDict() - for col in columns: - value = getattr(user, col) - if not value: - value = '' - if col == 'created': - value = str(value) # or maybe dd/mm/yyyy? - row[col] = value - row_dicts.append(row) - - writer = CsvWriter(row_dicts) - writer.save(dump_file_obj) - dump_file_obj.close() diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index 14b6874c3d1..65bc014900c 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -179,9 +179,21 @@ class IDomainObjectModification(Interface): """ def notify(self, entity, operation): + """ + Send a notification on entity modification. + + :param entity: instance of module.Package. + :param operation: 'new', 'changed' or 'deleted'. + """ pass def notify_after_commit(self, entity, operation): + """ + Send a notification after entity modification. + + :param entity: instance of module.Package. + :param operation: 'new', 'changed' or 'deleted'. + """ pass @@ -191,6 +203,11 @@ class IResourceUrlChange(Interface): """ def notify(self, resource): + """ + Give user a notify is resource url has changed. + + :param resource, instance of model.Resource + """ pass diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js index e6cb99cd31a..7fef17dfcc4 100644 --- a/ckan/public/base/javascript/modules/image-upload.js +++ b/ckan/public/base/javascript/modules/image-upload.js @@ -1,6 +1,6 @@ /* Image Upload - * - */ + * + */ this.ckan.module('image-upload', function($, _) { return { /* options object can be extended using data-module-* attributes */ @@ -10,6 +10,7 @@ this.ckan.module('image-upload', function($, _) { field_upload: 'image_upload', field_url: 'image_url', field_clear: 'clear_upload', + field_name: 'name', upload_label: '', i18n: { upload: _('Upload'), @@ -21,6 +22,12 @@ this.ckan.module('image-upload', function($, _) { } }, + /* Should be changed to true if user modifies resource's name + * + * @type {Boolean} + */ + _nameIsDirty: false, + /* Initialises the module setting up elements and event listeners. * * Returns nothing. @@ -33,11 +40,13 @@ this.ckan.module('image-upload', function($, _) { var field_upload = 'input[name="' + options.field_upload + '"]'; var field_url = 'input[name="' + options.field_url + '"]'; var field_clear = 'input[name="' + options.field_clear + '"]'; + var field_name = 'input[name="' + options.field_name + '"]'; this.input = $(field_upload, this.el); this.field_url = $(field_url, this.el).parents('.control-group'); this.field_image = this.input.parents('.control-group'); this.field_url_input = $('input', this.field_url); + this.field_name = this.el.parents('form').find(field_name); // Is there a clear checkbox on the form already? var checkbox = $(field_clear, this.el); @@ -85,6 +94,15 @@ this.ckan.module('image-upload', function($, _) { .add(this.field_url) .add(this.field_image); + // Disables autoName if user modifies name field + this.field_name + .on('change', this._onModifyName); + // Disables autoName if resource name already has value, + // i.e. we on edit page + if (this.field_name.val()){ + this._nameIsDirty = true; + } + if (options.is_url) { this._showOnlyFieldUrl(); } else if (options.is_upload) { @@ -101,7 +119,8 @@ this.ckan.module('image-upload', function($, _) { */ _onFromWeb: function() { this._showOnlyFieldUrl(); - this.field_url_input.focus(); + this.field_url_input.focus() + .on('blur', this._onFromWebBlur); if (this.options.is_upload) { this.field_clear.val('true'); } @@ -128,6 +147,7 @@ this.ckan.module('image-upload', function($, _) { this.field_url_input.prop('readonly', true); this.field_clear.val(''); this._showOnlyFieldUrl(); + this._autoName(file_name); }, /* Show only the buttons, hiding all others @@ -166,7 +186,36 @@ this.ckan.module('image-upload', function($, _) { */ _onInputMouseOut: function() { this.button_upload.removeClass('hover'); - } + }, + /* Event listener for changes in resource's name by direct input from user + * + * Returns nothing + */ + _onModifyName: function() { + this._nameIsDirty = true; + }, + + /* Event listener for when someone loses focus of URL field + * + * Returns nothing + */ + _onFromWebBlur: function() { + var url = this.field_url_input.val().match(/([^\/]+)\/?$/) + if (url) { + this._autoName(url.pop()); + } + }, + + /* Automatically add file name into field Name + * + * Select by attribute [name] to be on the safe side and allow to change field id + * Returns nothing + */ + _autoName: function(name) { + if (!this._nameIsDirty){ + this.field_name.val(name); + } + } }; }); diff --git a/ckan/public/base/test/index.html b/ckan/public/base/test/index.html index bdadaa48622..1c9317807f5 100644 --- a/ckan/public/base/test/index.html +++ b/ckan/public/base/test/index.html @@ -46,6 +46,7 @@ + @@ -59,6 +60,7 @@ + diff --git a/ckan/public/base/test/spec/modules/image-upload.spec.js b/ckan/public/base/test/spec/modules/image-upload.spec.js new file mode 100644 index 00000000000..f0f2939494d --- /dev/null +++ b/ckan/public/base/test/spec/modules/image-upload.spec.js @@ -0,0 +1,65 @@ +/*globals describe beforeEach afterEach it assert sinon ckan jQuery */ +describe('ckan.modules.ImageUploadModule()', function () { + var ImageUploadModule = ckan.module.registry['image-upload']; + + beforeEach(function () { + this.el = document.createElement('div'); + this.sandbox = ckan.sandbox(); + this.module = new ImageUploadModule(this.el, {}, this.sandbox); + this.module.el.html([ + '
', + '', + ]); + this.module.initialize(); + this.module.field_name = jQuery('', {type: 'text'}) + }); + + afterEach(function () { + this.module.teardown(); + }); + + describe('._onFromWeb()', function () { + + it('should change name when url changed', function () { + this.module.field_url_input.val('http://example.com/some_image.png'); + this.module._onFromWebBlur(); + assert.equal(this.module.field_name.val(), 'some_image.png'); + + this.module.field_url_input.val('http://example.com/undefined_file'); + this.module._onFromWebBlur(); + assert.equal(this.module.field_name.val(), 'undefined_file'); + }); + + it('should ignore url changes if name was manualy changed', function () { + this.module.field_url_input.val('http://example.com/some_image.png'); + this.module._onFromWebBlur(); + assert.equal(this.module.field_name.val(), 'some_image.png'); + + this.module._onModifyName(); + + this.module.field_url_input.val('http://example.com/undefined_file'); + this.module._onFromWebBlur(); + assert.equal(this.module.field_name.val(), 'some_image.png'); + }); + + it('should ignore url changes if name was filled before', function () { + this.module._nameIsDirty = true; + this.module.field_name.val('prefilled'); + + this.module.field_url_input.val('http://example.com/some_image.png'); + this.module._onFromWebBlur(); + assert.equal(this.module.field_name.val(), 'prefilled'); + + this.module.field_url_input.val('http://example.com/second_some_image.png'); + this.module._onFromWebBlur(); + assert.equal(this.module.field_name.val(), 'prefilled'); + + this.module._onModifyName() + + this.module.field_url_input.val('http://example.com/undefined_file'); + this.module._onFromWebBlur(); + assert.equal(this.module.field_name.val(), 'prefilled'); + }); + }); + +}); diff --git a/ckan/tests/controllers/test_user.py b/ckan/tests/controllers/test_user.py index b7e8fe7bdb8..fc3198ff74f 100644 --- a/ckan/tests/controllers/test_user.py +++ b/ckan/tests/controllers/test_user.py @@ -246,6 +246,37 @@ def test_edit_user(self): assert_equal(user.about, 'new about') assert_equal(user.activity_streams_email_notifications, True) + def test_email_change_without_password(self): + + app = self._get_test_app() + env, response, user = _get_user_edit_page(app) + + form = response.forms['user-edit-form'] + + # new values + form['email'] = 'new@example.com' + + # factory returns user with password 'pass' + form.fields['old_password'][0].value = 'wrong-pass' + + response = webtest_submit(form, 'save', status=200, extra_environ=env) + assert_true('Old Password: incorrect password' in response) + + def test_email_change_with_password(self): + app = self._get_test_app() + env, response, user = _get_user_edit_page(app) + + form = response.forms['user-edit-form'] + + # new values + form['email'] = 'new@example.com' + + # factory returns user with password 'pass' + form.fields['old_password'][0].value = 'pass' + + response = submit_and_follow(app, form, env, 'save') + assert_true('Profile updated' in response) + def test_perform_reset_for_key_change(self): password = 'password' params = {'password1': password, 'password2': password} diff --git a/ckan/tests/legacy/lib/test_cli.py b/ckan/tests/legacy/lib/test_cli.py index ce71774d658..33c093ae37c 100644 --- a/ckan/tests/legacy/lib/test_cli.py +++ b/ckan/tests/legacy/lib/test_cli.py @@ -10,45 +10,6 @@ from ckan.lib.search import index_for,query_for, clear_all -class TestDb: - @classmethod - def setup_class(cls): - cls.db = ManageDb('db') - CreateTestData.create() - - # delete warandpeace - rev = model.repo.new_revision() - model.Package.by_name(u'warandpeace').delete() - model.repo.commit_and_remove() - - @classmethod - def teardown_class(cls): - model.repo.rebuild_db() - - def test_simple_dump_csv(self): - csv_filepath = '/tmp/dump.tmp' - self.db.args = ('simple-dump-csv %s' % csv_filepath).split() - self.db.simple_dump_csv() - assert os.path.exists(csv_filepath), csv_filepath - f_obj = open(csv_filepath, "r") - reader = csv.reader(f_obj) - rows = [row for row in reader] - assert_equal(rows[0][:3], ['id', 'name', 'title']) - pkg_names = set(row[1] for row in rows[1:]) - assert 'annakarenina' in pkg_names, pkg_names - assert 'warandpeace' not in pkg_names, pkg_names - - def test_simple_dump_json(self): - json_filepath = '/tmp/dump.tmp' - self.db.args = ('simple-dump-json %s' % json_filepath).split() - self.db.simple_dump_json() - assert os.path.exists(json_filepath), json_filepath - f_obj = open(json_filepath, "r") - rows = json.loads(f_obj.read()) - assert set(rows[0].keys()) > set(('id', 'name', 'title')), rows[0].keys() - pkg_names = set(row['name'] for row in rows) - assert 'annakarenina' in pkg_names, pkg_names - assert 'warandpeace' not in pkg_names, pkg_names class FakeOptions(): def __init__(self,**kwargs): diff --git a/ckan/tests/legacy/test_dumper.py b/ckan/tests/legacy/test_dumper.py deleted file mode 100644 index 2fba83aabfe..00000000000 --- a/ckan/tests/legacy/test_dumper.py +++ /dev/null @@ -1,53 +0,0 @@ -import tempfile - -from ckan.tests.legacy import TestController, CreateTestData -import ckan.model as model -import ckan.lib.dumper as dumper -simple_dumper = dumper.SimpleDumper() - - -class TestSimpleDump(TestController): - - @classmethod - def setup_class(self): - model.repo.rebuild_db() - CreateTestData.create() - - @classmethod - def teardown_class(self): - model.Session.remove() - model.repo.rebuild_db() - - def test_simple_dump_csv(self): - dump_file = tempfile.TemporaryFile() - simple_dumper.dump(dump_file, 'csv') - dump_file.seek(0) - res = dump_file.read() - assert 'annakarenina' in res, res - assert 'tolstoy' in res, res - assert 'russian' in res, res - assert 'genre' in res, res - assert 'romantic novel' in res, res - assert 'datahub.io/download' in res, res - assert 'Index of the novel' in res, res - assert 'joeadmin' not in res, res - self.assert_correct_field_order(res) - - def test_simple_dump_json(self): - dump_file = tempfile.TemporaryFile() - simple_dumper.dump(dump_file, 'json') - dump_file.seek(0) - res = dump_file.read() - assert 'annakarenina' in res, res - assert '"russian"' in res, res - assert 'genre' in res, res - assert 'romantic novel' in res, res - assert 'joeadmin' not in res, res - self.assert_correct_field_order(res) - - def assert_correct_field_order(self, res): - correct_field_order = ('id', 'name', 'title', 'version', 'url') - field_position = [res.find('"%s"' % field) for field in correct_field_order] - field_position_sorted = field_position[:] - field_position_sorted.sort() - assert field_position == field_position_sorted, field_position diff --git a/ckan/tests/logic/action/test_get.py b/ckan/tests/logic/action/test_get.py index 290e44719cf..c8608b838a4 100644 --- a/ckan/tests/logic/action/test_get.py +++ b/ckan/tests/logic/action/test_get.py @@ -37,6 +37,21 @@ def foo(key, data, errors, context): eq(dataset2['new_field'], 'foo') + def test_package_show_with_custom_schema_return_default_schema(self): + dataset1 = factories.Dataset() + from ckan.logic.schema import default_show_package_schema + custom_schema = default_show_package_schema() + + def foo(key, data, errors, context): + data[key] = 'foo' + custom_schema['new_field'] = [foo] + + dataset2 = helpers.call_action('package_show', id=dataset1['id'], + use_default_schema=True, + context={'schema': custom_schema}) + + assert 'new_field' not in dataset2 + def test_package_show_is_lazy(self): dataset1 = factories.Dataset() @@ -1197,6 +1212,38 @@ def test_package_works_without_user_in_context(self): ''' logic.get_action('package_search')({}, dict(q='anything')) + def test_custom_schema_returned(self): + if not p.plugin_loaded('example_idatasetform'): + p.load('example_idatasetform') + + dataset1 = factories.Dataset(custom_text='foo') + + query = helpers.call_action('package_search', + q='id:{0}'.format(dataset1['id'])) + + eq(query['results'][0]['id'], dataset1['id']) + eq(query['results'][0]['custom_text'], 'foo') + + p.unload('example_idatasetform') + + def test_custom_schema_not_returned(self): + + if not p.plugin_loaded('example_idatasetform'): + p.load('example_idatasetform') + + dataset1 = factories.Dataset(custom_text='foo') + + query = helpers.call_action('package_search', + q='id:{0}'.format(dataset1['id']), + use_default_schema=True) + + eq(query['results'][0]['id'], dataset1['id']) + assert 'custom_text' not in query['results'][0] + eq(query['results'][0]['extras'][0]['key'], 'custom_text') + eq(query['results'][0]['extras'][0]['value'], 'foo') + + p.unload('example_idatasetform') + class TestBadLimitQueryParameters(helpers.FunctionalTestBase): '''test class for #1258 non-int query parameters cause 500 errors diff --git a/doc/contributing/i18n.rst b/doc/contributing/i18n.rst index bb1af88afb2..e1fc1ee8e6b 100644 --- a/doc/contributing/i18n.rst +++ b/doc/contributing/i18n.rst @@ -30,16 +30,16 @@ If your language is present, you can switch the default language simply by setti If your language is not supported yet, the remainder of this section section provides instructions on how to prepare a translation file and add it to CKAN. ---------------------- -Adding a new language ---------------------- +---------------------------------------------------------- +Adding a new language or improving an existing translation +---------------------------------------------------------- -If you want to add an entirely new language to CKAN, you have two options. +If you want to add an entirely new language to CKAN or update an existing translation, you have two options. -* :ref:`i18n-transifex`. Creating translation files using Transifex, the open source translation software. +* :ref:`i18n-transifex`. Creating or updating translation files using Transifex, the open source translation software. To add a language you need to request it from the Transifex dashboard: https://www.transifex.com/okfn/ckan/dashboard/ Alternatively to update an existing language you need to request to join the appropriate CKAN language team. If you don't hear back from the CKAN administrators, contact them via the ckan-dev list. * :ref:`i18n-manual`. Creating translation files manually in your own branch. -.. note:: Translating CKAN in Transifex is only enabled when a 'call for translations' is issued. +.. note:: If you choose not to contribute your translation back via Transifex then you must ensure you make it public in another way, as per the requirements of CKAN's AGPL license. .. _i18n-transifex: @@ -51,24 +51,15 @@ Transifex, the open translation platform, provides a simple web interface for wr Using Transifex makes it easier to handle collaboration, with an online editor that makes the process more accessible. -Existing CKAN translation projects can be found at: https://www.transifex.net/projects/p/ckan/teams/ +Existing CKAN translation projects can be found at: https://www.transifex.com/okfn/ckan/content/ -When leading up to a CKAN release, the strings are loaded onto Transifex and ckan-discuss list is emailed to encourage translation work. When the release is done, the latest translations on Transifex are checked back into CKAN. +When leading up to a CKAN release, the strings are loaded onto Transifex and ckan-dev list is emailed to encourage translation work. When the release is done, the latest translations on Transifex are checked back into CKAN. Transifex administration ------------------------ -The Transifex workflow is as follows: - -* Install transifex command-line utilities -* ``tx init`` in CKAN to connect to Transifex -* Run ``python setup.py extract_messages`` on the CKAN source -* Upload the local .pot file via command-line ``tx push`` -* Get people to complete translations on Transifex -* Pull locale .po files via ``tx pull`` -* ``python setup.py compile_catalog`` -* Git Commit and push po and mo files +The Transifex workflow is described in the :doc:`release-process` .. _i18n-manual: @@ -82,21 +73,27 @@ All the English strings in CKAN are extracted into the ``ckan.pot`` file, which .. note:: For information, the pot file was created with the ``babel`` command ``python setup.py extract_messages``. -0. Install Babel +1. Preparation +-------------- + +This tutorial assumes you've got ckan installed as source in a virtualenv. Activate the virtualenv and cd to the ckan directory: + + .. parsed-literal:: + + |activate| + cd |virtualenv|/src/ckan + +2. Install Babel ---------------- You need Python's ``babel`` library (Debian package ``python-pybabel``). Install it as follows with pip:: - pip -E pyenv install babel + pip install --upgrade Babel -1. Create a 'po' file for your language +3. Create a 'po' file for your language --------------------------------------- -First, grab the CKAN i18n repository:: - - hg clone http://bitbucket.org/bboissin/ckan-i18n/ - -Then create a translation file for your language (a po file) using the pot file:: +Then create a translation file for your language (a po file) using the pot file (containing all the English strings):: python setup.py init_catalog --locale YOUR_LANGUAGE @@ -116,13 +113,12 @@ We recommend using a translation tool, such as `poedit ` 3. Commit the translation ------------------------- -When the po is complete, create a branch in your source, then commit it to the CKAN i18n repo:: - - git checkout master - git branch translation-YOUR_LANGUAGE +When the po is complete, create a branch in your source, then commit it to your own fork of the CKAN repo:: + git add ckan/i18n/YOUR_LANGUAGE/LC_MESSAGES/ckan.po git commit -m '[i18n]: New language po added: YOUR_LANGUAGE' ckan/i18n/YOUR_LANGUAGE/LC_MESSAGES/ckan.po - git push origin translation-YOUR_LANGUAGE + +NB it is not appropriate to do a Pull Request to the main ckan repo, since that takes its translations from Transifex. 4. Compile a translation ------------------------ @@ -142,19 +138,15 @@ This will result in a binary 'mo' file of your translation at ``ckan/i18n/YOUR_L 5. (optional) Deploy the translation ------------------------------------ -This section explains how to deploy your translation automatically to your host, if you are using a remote host. +This section explains how to deploy your translation to your CKAN server. -It assumes a standard layout on the server (you may want to check before you upload!) and that you are deploying to ``hu.ckan.net`` for language ``hu``. +Once you have a compiled translation file, copy it to your host: -Once you have a compiled translation file, for automated deployment to your host do:: + .. parsed-literal:: - fab config_0:hu.ckan.net upload_i18n:hu + scp ckan.mo |virtualenv|/src/ckan/ckan/i18n/hu/LC_MESSAGES/ckan.mo -See the ``config_0`` options if more configuration is needed e.g. of host or location. - -Alternatively, if you do not want to use fab, you can just scp:: - - scp ckan.mo /home/okfn/var/srvc/hu.ckan.net/pyenv/src/ckan/ckan/i18n/hu/LC_MESSAGES/ckan.mo +Adjust the path if you did not use the default location. This example is for language ``hu``. 6. Configure the language ------------------------- @@ -172,28 +164,13 @@ translations or submit a new one. At the same time we need to ensure the stability between CKAN releases, so the following guidelines apply when managing translations: -* Translations are open on Transifex as soon as a release branch for a minor - version is created. At this point the strings are not yet freezed and can - still change, but hopefully not too much. An announcement email will be sent - to the mailing list and via Transifex. - -* 2-3 weeks before the actual release the strings are frozen. This means that - no new strings will be added on this release line. A second announce email is - sent at this point. - -* The translations will be kept open on Transifex after the release, and will be - updated on each patch release, provided that: - - - They pass a review like any other change merged into a patch release (ie - they must not introducde backwards incompatible changes). - - Big changes like whole new languages (specially ones that introduce - new features like RTL supporti, etc) should be tested and reviewed on a - separate branch first. - - Ultimately is up to the commiters and the release manager to decide if a - new or updated translation is included in a patch release or needs to - wait until the next minor release. - -* The *master* branch is not currently translated. Translations from the latest - stable line (see :ref:`releases`) are cherry-picked into master after each - minor or patch release. +* About 3 weeks before a CKAN release, CKAN is branched, and the English + strings are frozen, and an announcement is made on ckan-dev to call for + translation work. They are given 2 weeks to translate any new strings in this + release. + +* During this period, translation is done on a 'resource' on Transifex which is + named to match the new CKAN version. It has been created as a copy of the + next most recent resource, so any new languages create or other updates done + on Transifex since the last release automatically go into the new release. diff --git a/doc/maintaining/configuration.rst b/doc/maintaining/configuration.rst index acd3f9e3c5d..8e444c24bfd 100644 --- a/doc/maintaining/configuration.rst +++ b/doc/maintaining/configuration.rst @@ -1572,14 +1572,14 @@ Default value: (none) By default, the locales are searched for in the ``ckan/i18n`` directory. Use this option if you want to use another folder. -.. _ckan.18n.extra_directory: +.. _ckan.i18n.extra_directory: ckan.i18n.extra_directory ^^^^^^^^^^^^^^^^^^^^^^^^^ Example:: - ckan.18n.extra_directory = /opt/ckan/extra_translations/ + ckan.i18n.extra_directory = /opt/ckan/extra_translations/ Default value: (none) @@ -1587,14 +1587,14 @@ If you wish to add extra translation strings and have them merged with the default ckan translations at runtime you can specify the location of the extra translations using this option. -.. _ckan.18n.extra_gettext_domain: +.. _ckan.i18n.extra_gettext_domain: ckan.i18n.extra_gettext_domain ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example:: - ckan.18n.extra_gettext_domain = mydomain + ckan.i18n.extra_gettext_domain = mydomain Default value: (none) @@ -1603,18 +1603,18 @@ example if your translations are stored as ``i18n//LC_MESSAGES/somedomain.mo`` you would want to set this option to ``somedomain`` -.. _ckan.18n.extra_locales: +.. _ckan.i18n.extra_locales: -ckan.18n.extra_locales -^^^^^^^^^^^^^^^^^^^^^^ +ckan.i18n.extra_locales +^^^^^^^^^^^^^^^^^^^^^^^ Example:: - ckan.18n.extra_locales = fr es de + ckan.i18n.extra_locales = fr es de Default value: (none) -If you have set an extra i18n directory using ``ckan.18n.extra_directory``, you +If you have set an extra i18n directory using ``ckan.i18n.extra_directory``, you should specify the locales that have been translated in that directory in this option. diff --git a/doc/maintaining/paster.rst b/doc/maintaining/paster.rst index 653565f839b..d7490ad2da3 100644 --- a/doc/maintaining/paster.rst +++ b/doc/maintaining/paster.rst @@ -330,21 +330,15 @@ data in the database!) and then load the file: includes API keys and other user data which may be regarded as private. So keep it secure, like your database server. -Exporting Datasets to JSON or CSV ---------------------------------- +Exporting Datasets to JSON Lines +-------------------------------- -You can export all of your CKAN site's datasets from your database to a JSON file -using the ``db simple-dump-json`` command: +You can export all of your CKAN site's datasets from your database to a JSON +Lines file using the ``ckanapi dump datasets`` command: .. parsed-literal:: - paster db simple-dump-json -c |production.ini| my_datasets.json - -To export the datasets in CSV format instead, use ``db simple-dump-csv``: - -.. parsed-literal:: - - paster db simple-dump-csv -c |production.ini| my_datasets.csv + ckanapi dump datasets -c |production.ini| -O my_datasets.jsonl This is useful to create a simple public listing of the datasets, with no user information. Some simple additions to the Apache config can serve the dump @@ -365,15 +359,15 @@ virtual Apache config file (e.g. |apache_config_file|):: dump`` command), as those contain private user information such as email addresses and API keys. -Exporting User Accounts to CSV ------------------------------- +Exporting User Accounts to JSON Lines +------------------------------------- -You can export all of your CKAN site's user accounts from your database to a CSV file -using the ``db user-dump-csv`` command: +You can export all of your CKAN site's user accounts from your database to +a JSON Lines file using the ``ckanapi dump users`` command: .. parsed-literal:: - paster db user-dump-csv -c |production.ini| my_database_users.csv + ckanapi dump users -c |production.ini| -O my_database_users.jsonl front-end-build: Creates and minifies css and JavaScript files ==============================================================