From 1d921ba3758ebd577db05ed5a447d30ab021a196 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 10 Sep 2019 10:57:49 +0300 Subject: [PATCH 1/9] ResourceProxy Controller -> Blueprint --- ckanext/resourceproxy/blueprint.py | 112 ++++++++++++++++++++++ ckanext/resourceproxy/plugin.py | 47 +++++---- ckanext/resourceproxy/tests/test_proxy.py | 9 +- 3 files changed, 144 insertions(+), 24 deletions(-) create mode 100644 ckanext/resourceproxy/blueprint.py diff --git a/ckanext/resourceproxy/blueprint.py b/ckanext/resourceproxy/blueprint.py new file mode 100644 index 00000000000..affdec16a3b --- /dev/null +++ b/ckanext/resourceproxy/blueprint.py @@ -0,0 +1,112 @@ +# encoding: utf-8 + +from logging import getLogger + +import requests +import urlparse +from flask import Blueprint, make_response + +import ckan.lib.base as base +import ckan.logic as logic +from ckan.common import config, _ +from ckan.plugins.toolkit import (asint, abort, get_action, c) + +log = getLogger(__name__) + +MAX_FILE_SIZE = asint( + config.get(u'ckan.resource_proxy.max_file_size', 1024**2) +) +CHUNK_SIZE = asint(config.get(u'ckan.resource_proxy.chunk_size', 4096)) + +resource_proxy = Blueprint(u'resource_proxy', __name__) + + +def proxy_resource(context, data_dict): + u'''Chunked proxy for resources. To make sure that the file is not too + large, first, we try to get the content length from the headers. + If the headers to not contain a content length (if it is a chinked + response), we only transfer as long as the transferred data is + less than the maximum file size. + + ''' + resource_id = data_dict[u'resource_id'] + log.info(u'Proxify resource {id}'.format(id=resource_id)) + try: + resource = get_action(u'resource_show')(context, {u'id': resource_id}) + except logic.NotFound: + return abort(404, _(u'Resource not found')) + url = resource[u'url'] + + parts = urlparse.urlsplit(url) + if not parts.scheme or not parts.netloc: + return abort(409, _(u'Invalid URL.')) + + response = make_response() + try: + # first we try a HEAD request which may not be supported + did_get = False + r = requests.head(url) + # Servers can refuse HEAD requests. 405 is the appropriate + # response, but 400 with the invalid method mentioned in the + # text, or a 403 (forbidden) status is also possible (#2412, + # #2530) + if r.status_code in (400, 403, 405): + r = requests.get(url, stream=True) + did_get = True + r.raise_for_status() + + cl = r.headers.get(u'content-length') + if cl and int(cl) > MAX_FILE_SIZE: + return abort( + 409, ( + u'Content is too large to be proxied. Allowed' + u'file size: {allowed}, Content-Length: {actual}.' + ).format(allowed=MAX_FILE_SIZE, actual=cl) + ) + + if not did_get: + r = requests.get(url, stream=True) + + response.headers[u'content-type'] = r.headers[u'content-type'] + response.charset = r.encoding + + length = 0 + for chunk in r.iter_content(chunk_size=CHUNK_SIZE): + response.stream.write(chunk) + length += len(chunk) + + if length >= MAX_FILE_SIZE: + return abort( + 409, + headers={u'content-encoding': u''}, + detail=u'Content is too large to be proxied.' + ) + + except requests.exceptions.HTTPError as error: + details = u'Could not proxy resource. Server responded with %s %s' % ( + error.response.status_code, error.response.reason + ) + return abort(409, detail=details) + except requests.exceptions.ConnectionError as error: + details = u'''Could not proxy resource because a + connection error occurred. %s''' % error + return abort(502, detail=details) + except requests.exceptions.Timeout: + details = u'Could not proxy resource because the connection timed out.' + return abort(504, detail=details) + return response + + +def proxy_view(id, resource_id): + data_dict = {u'resource_id': resource_id} + context = { + u'model': base.model, + u'session': base.model.Session, + u'user': c.user + } + return proxy_resource(context, data_dict) + + +resource_proxy.add_url_rule( + u'/dataset//resource//proxy', view_func=proxy_view +) diff --git a/ckanext/resourceproxy/plugin.py b/ckanext/resourceproxy/plugin.py index edfd34aa8a8..b0c473c3d6c 100644 --- a/ckanext/resourceproxy/plugin.py +++ b/ckanext/resourceproxy/plugin.py @@ -8,10 +8,12 @@ import urlparse from ckan.common import config +from ckanext.resourceproxy import blueprint + log = getLogger(__name__) -def get_proxified_resource_url(data_dict, proxy_schemes=['http','https']): +def get_proxified_resource_url(data_dict, proxy_schemes=['http', 'https']): ''' :param data_dict: contains a resource and package dict :type data_dict: dictionary @@ -25,10 +27,10 @@ def get_proxified_resource_url(data_dict, proxy_schemes=['http','https']): compare_domains = datapreview.compare_domains if not compare_domains([ckan_url, url]) and scheme in proxy_schemes: url = h.url_for( - action='proxy_resource', - controller='ckanext.resourceproxy.controller:ProxyController', + 'resource_proxy.proxy_view', id=data_dict['package']['name'], - resource_id=data_dict['resource']['id']) + resource_id=data_dict['resource']['id'] + ) log.info('Proxified url is {0}'.format(url)) return url @@ -46,30 +48,35 @@ class ResourceProxy(p.SingletonPlugin): 1. Import the proxy plugin if it exists ``import ckanext.resourceproxy.plugin as proxy`` - 2. In you extension, make sure that the proxy plugin is - enabled by checking the ``ckan.resource_proxy_enabled`` config variable. + 2. In you extension, make sure that the proxy plugin is enabled by + checking the ``ckan.resource_proxy_enabled`` config variable. ``config.get('ckan.resource_proxy_enabled', False)`` + """ - p.implements(p.IRoutes, inherit=True) p.implements(p.ITemplateHelpers, inherit=True) + p.implements(p.IBlueprint) - - def before_map(self, m): - m.connect('/dataset/{id}/resource/{resource_id}/proxy', - controller='ckanext.resourceproxy.controller:ProxyController', - action='proxy_resource') - return m + def get_blueprint(self): + return blueprint.resource_proxy def get_helpers(self): return {'view_resource_url': self.view_resource_url} - def view_resource_url(self, resource_view, resource, - package, proxy_schemes=['http','https']): + def view_resource_url( + self, + resource_view, + resource, + package, + proxy_schemes=['http', 'https'] + ): ''' Returns the proxy url if its availiable ''' - data_dict = {'resource_view': resource_view, - 'resource': resource, - 'package': package} - return get_proxified_resource_url(data_dict, - proxy_schemes=proxy_schemes) + data_dict = { + 'resource_view': resource_view, + 'resource': resource, + 'package': package + } + return get_proxified_resource_url( + data_dict, proxy_schemes=proxy_schemes + ) diff --git a/ckanext/resourceproxy/tests/test_proxy.py b/ckanext/resourceproxy/tests/test_proxy.py index 2d062433a54..5fad792d9ce 100644 --- a/ckanext/resourceproxy/tests/test_proxy.py +++ b/ckanext/resourceproxy/tests/test_proxy.py @@ -12,7 +12,7 @@ import ckan.model as model import ckan.plugins as p import ckan.lib.create_test_data as create_test_data -import ckanext.resourceproxy.controller as controller +import ckanext.resourceproxy.blueprint as blueprint import ckanext.resourceproxy.plugin as proxy @@ -53,10 +53,11 @@ class TestProxyPrettyfied(unittest.TestCase): @classmethod def setup_class(cls): cls._original_config = config.copy() - cls.app = _get_test_app() if not p.plugin_loaded('resource_proxy'): p.load('resource_proxy') config['ckan.plugins'] = 'resource_proxy' + cls.app = _get_test_app() + create_test_data.CreateTestData.create() @classmethod @@ -106,7 +107,7 @@ def test_resource_proxy_on_404(self): @responses.activate def test_large_file(self): - cl = controller.MAX_FILE_SIZE + 1 + cl = blueprint.MAX_FILE_SIZE + 1 self.mock_out_urls( self.url, headers={'Content-Length': six.text_type(cl)}, @@ -119,7 +120,7 @@ def test_large_file(self): @responses.activate def test_large_file_streaming(self): - cl = controller.MAX_FILE_SIZE + 1 + cl = blueprint.MAX_FILE_SIZE + 1 self.mock_out_urls( self.url, stream=True, From b08db790406733346cffc112a50ec5070f63235c Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 10 Sep 2019 11:10:29 +0300 Subject: [PATCH 2/9] Drop resourceproxy/plugin from pep8 blacklist --- ckan/tests/legacy/test_coding_standards.py | 1 - ckanext/resourceproxy/plugin.py | 28 +++++++++++----------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/ckan/tests/legacy/test_coding_standards.py b/ckan/tests/legacy/test_coding_standards.py index be0137bc657..00628fc9eb0 100644 --- a/ckan/tests/legacy/test_coding_standards.py +++ b/ckan/tests/legacy/test_coding_standards.py @@ -467,7 +467,6 @@ class TestPep8(object): 'ckanext/example_idatasetform/plugin.py', 'ckanext/example_itemplatehelpers/plugin.py', 'ckanext/multilingual/plugin.py', - 'ckanext/resourceproxy/plugin.py', 'ckanext/stats/controller.py', 'ckanext/stats/plugin.py', 'ckanext/stats/stats.py', diff --git a/ckanext/resourceproxy/plugin.py b/ckanext/resourceproxy/plugin.py index b0c473c3d6c..eecc2182ce9 100644 --- a/ckanext/resourceproxy/plugin.py +++ b/ckanext/resourceproxy/plugin.py @@ -13,25 +13,25 @@ log = getLogger(__name__) -def get_proxified_resource_url(data_dict, proxy_schemes=['http', 'https']): - ''' +def get_proxified_resource_url(data_dict, proxy_schemes=[u'http', u'https']): + u''' :param data_dict: contains a resource and package dict :type data_dict: dictionary :param proxy_schemes: list of url schemes to proxy for. :type data_dict: list ''' - ckan_url = config.get('ckan.site_url', '//localhost:5000') - url = data_dict['resource']['url'] + ckan_url = config.get(u'ckan.site_url', u'//localhost:5000') + url = data_dict[u'resource'][u'url'] scheme = urlparse.urlparse(url).scheme compare_domains = datapreview.compare_domains if not compare_domains([ckan_url, url]) and scheme in proxy_schemes: url = h.url_for( - 'resource_proxy.proxy_view', - id=data_dict['package']['name'], - resource_id=data_dict['resource']['id'] + u'resource_proxy.proxy_view', + id=data_dict[u'package'][u'name'], + resource_id=data_dict[u'resource'][u'id'] ) - log.info('Proxified url is {0}'.format(url)) + log.info(u'Proxified url is {0}'.format(url)) return url @@ -60,22 +60,22 @@ def get_blueprint(self): return blueprint.resource_proxy def get_helpers(self): - return {'view_resource_url': self.view_resource_url} + return {u'view_resource_url': self.view_resource_url} def view_resource_url( self, resource_view, resource, package, - proxy_schemes=['http', 'https'] + proxy_schemes=[u'http', u'https'] ): - ''' + u''' Returns the proxy url if its availiable ''' data_dict = { - 'resource_view': resource_view, - 'resource': resource, - 'package': package + u'resource_view': resource_view, + u'resource': resource, + u'package': package } return get_proxified_resource_url( data_dict, proxy_schemes=proxy_schemes From a1564a6ec3315065744a46055ab231a4fd96087d Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 10 Sep 2019 12:15:22 +0300 Subject: [PATCH 3/9] Do not modify resource url if proxy not enabled --- ckanext/resourceproxy/plugin.py | 4 +++- ckanext/textview/tests/test_view.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ckanext/resourceproxy/plugin.py b/ckanext/resourceproxy/plugin.py index eecc2182ce9..4742dcef166 100644 --- a/ckanext/resourceproxy/plugin.py +++ b/ckanext/resourceproxy/plugin.py @@ -20,9 +20,11 @@ def get_proxified_resource_url(data_dict, proxy_schemes=[u'http', u'https']): :param proxy_schemes: list of url schemes to proxy for. :type data_dict: list ''' + url = data_dict[u'resource'][u'url'] + if not p.plugin_loaded(u'resource_proxy'): + return url ckan_url = config.get(u'ckan.site_url', u'//localhost:5000') - url = data_dict[u'resource'][u'url'] scheme = urlparse.urlparse(url).scheme compare_domains = datapreview.compare_domains if not compare_domains([ckan_url, url]) and scheme in proxy_schemes: diff --git a/ckanext/textview/tests/test_view.py b/ckanext/textview/tests/test_view.py index 87cd7423606..37b700bf2e5 100644 --- a/ckanext/textview/tests/test_view.py +++ b/ckanext/textview/tests/test_view.py @@ -105,7 +105,6 @@ def test_js_included(self): id=self.package.name, resource_id=self.resource_id, view_id=self.resource_view['id']) result = app.get(url) - print(result.body) assert (('text_view.js' in result.body) or # Source file ('textview.js' in result.body)) # Compiled file # Restore the config to its original values From 0fe14b220792f7ca256af77335f9ee34801e6fe6 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 10 Sep 2019 13:04:21 +0300 Subject: [PATCH 4/9] Prepare urlsplit to py3 --- ckanext/resourceproxy/blueprint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/resourceproxy/blueprint.py b/ckanext/resourceproxy/blueprint.py index affdec16a3b..61f5661ef6e 100644 --- a/ckanext/resourceproxy/blueprint.py +++ b/ckanext/resourceproxy/blueprint.py @@ -3,7 +3,7 @@ from logging import getLogger import requests -import urlparse +from six.moves.urllib.parse import urlsplit from flask import Blueprint, make_response import ckan.lib.base as base @@ -37,7 +37,7 @@ def proxy_resource(context, data_dict): return abort(404, _(u'Resource not found')) url = resource[u'url'] - parts = urlparse.urlsplit(url) + parts = urlsplit(url) if not parts.scheme or not parts.netloc: return abort(409, _(u'Invalid URL.')) From c5654ba6dc6d5a6bafb90731df0ddf070f801cbf Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Fri, 13 Sep 2019 19:32:49 +0300 Subject: [PATCH 5/9] Remove resourceproxy controller --- ckanext/resourceproxy/controller.py | 91 ----------------------------- 1 file changed, 91 deletions(-) delete mode 100644 ckanext/resourceproxy/controller.py diff --git a/ckanext/resourceproxy/controller.py b/ckanext/resourceproxy/controller.py deleted file mode 100644 index 158044c3005..00000000000 --- a/ckanext/resourceproxy/controller.py +++ /dev/null @@ -1,91 +0,0 @@ -# encoding: utf-8 - -from logging import getLogger - -from six.moves.urllib.parse import urlsplit -import requests - -from ckan.common import config - -import ckan.logic as logic -import ckan.lib.base as base -from ckan.common import _ -from ckan.plugins.toolkit import asint - -log = getLogger(__name__) - -MAX_FILE_SIZE = asint(config.get('ckan.resource_proxy.max_file_size', 1024**2)) -CHUNK_SIZE = asint(config.get('ckan.resource_proxy.chunk_size', 4096)) - - -def proxy_resource(context, data_dict): - ''' Chunked proxy for resources. To make sure that the file is not too - large, first, we try to get the content length from the headers. - If the headers to not contain a content length (if it is a chinked - response), we only transfer as long as the transferred data is less - than the maximum file size. ''' - resource_id = data_dict['resource_id'] - log.info('Proxify resource {id}'.format(id=resource_id)) - try: - resource = logic.get_action('resource_show')(context, {'id': - resource_id}) - except logic.NotFound: - base.abort(404, _('Resource not found')) - url = resource['url'] - - parts = urlsplit(url) - if not parts.scheme or not parts.netloc: - base.abort(409, detail='Invalid URL.') - - try: - # first we try a HEAD request which may not be supported - did_get = False - r = requests.head(url) - # Servers can refuse HEAD requests. 405 is the appropriate response, - # but 400 with the invalid method mentioned in the text, or a 403 - # (forbidden) status is also possible (#2412, #2530) - if r.status_code in (400, 403, 405): - r = requests.get(url, stream=True) - did_get = True - r.raise_for_status() - - cl = r.headers.get('content-length') - if cl and int(cl) > MAX_FILE_SIZE: - base.abort(409, '''Content is too large to be proxied. Allowed - file size: {allowed}, Content-Length: {actual}.'''.format( - allowed=MAX_FILE_SIZE, actual=cl)) - - if not did_get: - r = requests.get(url, stream=True) - - base.response.content_type = r.headers['content-type'] - base.response.charset = r.encoding - - length = 0 - for chunk in r.iter_content(chunk_size=CHUNK_SIZE): - base.response.body_file.write(chunk) - length += len(chunk) - - if length >= MAX_FILE_SIZE: - base.abort(409, headers={'content-encoding': ''}, - detail='Content is too large to be proxied.') - - except requests.exceptions.HTTPError as error: - details = 'Could not proxy resource. Server responded with %s %s' % ( - error.response.status_code, error.response.reason) - base.abort(409, detail=details) - except requests.exceptions.ConnectionError as error: - details = '''Could not proxy resource because a - connection error occurred. %s''' % error - base.abort(502, detail=details) - except requests.exceptions.Timeout as error: - details = 'Could not proxy resource because the connection timed out.' - base.abort(504, detail=details) - - -class ProxyController(base.BaseController): - def proxy_resource(self, resource_id): - data_dict = {'resource_id': resource_id} - context = {'model': base.model, 'session': base.model.Session, - 'user': base.c.user} - return proxy_resource(context, data_dict) From 31852d254c3671f76569d6f2334ca1b0fd981339 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Fri, 13 Sep 2019 19:33:20 +0300 Subject: [PATCH 6/9] Remove datastore controller --- ckanext/datastore/controller.py | 209 -------------------------------- 1 file changed, 209 deletions(-) delete mode 100644 ckanext/datastore/controller.py diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py deleted file mode 100644 index 022490b891c..00000000000 --- a/ckanext/datastore/controller.py +++ /dev/null @@ -1,209 +0,0 @@ -# encoding: utf-8 - -from ckan.plugins.toolkit import ( - Invalid, - ObjectNotFound, - NotAuthorized, - get_action, - get_validator, - _, - request, - response, - BaseController, - abort, - render, - c, - h, - config, -) -from ckanext.datastore.writer import ( - csv_writer, - tsv_writer, - json_writer, - xml_writer, -) -from ckanext.datastore.logic.schema import ( - list_of_strings_or_string, - json_validator, - unicode_or_json_validator, -) -from ckan.logic import ( - tuplize_dict, - parse_params, -) -import ckan.lib.navl.dictization_functions as dict_fns - -from itertools import izip_longest - -int_validator = get_validator('int_validator') -boolean_validator = get_validator('boolean_validator') -ignore_missing = get_validator('ignore_missing') -OneOf = get_validator('OneOf') -default = get_validator('default') -unicode_only = get_validator('unicode_only') - - -DUMP_FORMATS = 'csv', 'tsv', 'json', 'xml' -PAGINATE_BY = 32000 - - -def dump_schema(): - return { - 'offset': [default(0), int_validator], - 'limit': [ignore_missing, int_validator], - 'format': [default('csv'), OneOf(DUMP_FORMATS)], - 'bom': [default(False), boolean_validator], - 'filters': [ignore_missing, json_validator], - 'q': [ignore_missing, unicode_or_json_validator], - 'distinct': [ignore_missing, boolean_validator], - 'plain': [ignore_missing, boolean_validator], - 'language': [ignore_missing, unicode_only], - 'fields': [ignore_missing, list_of_strings_or_string], - 'sort': [default('_id'), list_of_strings_or_string], - } - - -class DatastoreController(BaseController): - def dump(self, resource_id): - data, errors = dict_fns.validate(dict(request.GET), dump_schema()) - if errors: - abort(400, u'\n'.join( - u'{0}: {1}'.format(k, ' '.join(e)) for k, e in errors.items())) - - try: - dump_to( - resource_id, - response, - fmt=data['format'], - offset=data['offset'], - limit=data.get('limit'), - options={u'bom': data['bom']}, - sort=data['sort'], - search_params={ - k: v for k, v in data.items() if k in [ - 'filters', - 'q', - 'distinct', - 'plain', - 'language', - 'fields']}, - ) - except ObjectNotFound: - abort(404, _('DataStore resource not found')) - - def dictionary(self, id, resource_id): - u'''data dictionary view: show/edit field labels and descriptions''' - - try: - # resource_edit_base template uses these - c.pkg_dict = get_action('package_show')( - None, {'id': id}) - c.resource = get_action('resource_show')( - None, {'id': resource_id}) - rec = get_action('datastore_search')(None, { - 'resource_id': resource_id, - 'limit': 0}) - except (ObjectNotFound, NotAuthorized): - abort(404, _('Resource not found')) - - fields = [f for f in rec['fields'] if not f['id'].startswith('_')] - - if request.method == 'POST': - data = dict_fns.unflatten(tuplize_dict(parse_params( - request.params))) - info = data.get(u'info') - if not isinstance(info, list): - info = [] - info = info[:len(fields)] - - get_action('datastore_create')(None, { - 'resource_id': resource_id, - 'force': True, - 'fields': [{ - 'id': f['id'], - 'type': f['type'], - 'info': fi if isinstance(fi, dict) else {} - } for f, fi in izip_longest(fields, info)]}) - - h.flash_success(_('Data Dictionary saved. Any type overrides will ' - 'take effect when the resource is next uploaded ' - 'to DataStore')) - h.redirect_to( - controller='ckanext.datastore.controller:DatastoreController', - action='dictionary', - id=id, - resource_id=resource_id) - - return render( - 'datastore/dictionary.html', - extra_vars={ - 'fields': fields, - 'pkg_dict': c.pkg_dict, - 'resource': c.resource, - }) - - -def dump_to( - resource_id, output, fmt, offset, limit, options, sort, search_params): - if fmt == 'csv': - writer_factory = csv_writer - records_format = 'csv' - elif fmt == 'tsv': - writer_factory = tsv_writer - records_format = 'tsv' - elif fmt == 'json': - writer_factory = json_writer - records_format = 'lists' - elif fmt == 'xml': - writer_factory = xml_writer - records_format = 'objects' - - def start_writer(fields): - bom = options.get(u'bom', False) - return writer_factory(output, fields, resource_id, bom) - - def result_page(offs, lim): - return get_action('datastore_search')(None, dict({ - 'resource_id': resource_id, - 'limit': - PAGINATE_BY if limit is None - else min(PAGINATE_BY, lim), - 'offset': offs, - 'sort': sort, - 'records_format': records_format, - 'include_total': False, - }, **search_params)) - - result = result_page(offset, limit) - - if result['limit'] != limit: - # `limit` (from PAGINATE_BY) must have been more than - # ckan.datastore.search.rows_max, so datastore_search responded with a - # limit matching ckan.datastore.search.rows_max. So we need to paginate - # by that amount instead, otherwise we'll have gaps in the records. - paginate_by = result['limit'] - else: - paginate_by = PAGINATE_BY - - with start_writer(result['fields']) as wr: - while True: - if limit is not None and limit <= 0: - break - - records = result['records'] - - wr.write_records(records) - - if records_format == 'objects' or records_format == 'lists': - if len(records) < paginate_by: - break - elif not records: - break - - offset += paginate_by - if limit is not None: - limit -= paginate_by - if limit <= 0: - break - - result = result_page(offset, limit) From f0c4f837fabcb87ccfbb45ea9471c365fbfe7dbb Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Fri, 13 Sep 2019 20:16:52 +0300 Subject: [PATCH 7/9] Fix tests --- ckan/tests/test_coding_standards.py | 2 -- ckanext/datastore/tests/test_dump.py | 14 +++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py index 944c57ca809..980a54280f4 100644 --- a/ckan/tests/test_coding_standards.py +++ b/ckan/tests/test_coding_standards.py @@ -583,7 +583,6 @@ def find_unprefixed_string_literals(filename): u'ckanext/datapusher/tests/test_action.py', u'ckanext/datapusher/tests/test_default_views.py', u'ckanext/datapusher/tests/test_interfaces.py', - u'ckanext/datastore/controller.py', u'ckanext/datastore/helpers.py', u'ckanext/datastore/backend/postgres.py', u'ckanext/datastore/interfaces.py', @@ -612,7 +611,6 @@ def find_unprefixed_string_literals(filename): u'ckanext/example_iauthfunctions/plugin_v5_custom_config_setting.py', u'ckanext/example_iauthfunctions/plugin_v6_parent_auth_functions.py', u'ckanext/example_iauthfunctions/tests/test_example_iauthfunctions.py', - u'ckanext/example_iconfigurer/controller.py', u'ckanext/example_iconfigurer/plugin.py', u'ckanext/example_iconfigurer/plugin_v1.py', u'ckanext/example_iconfigurer/plugin_v2.py', diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index bbe52ebb6ac..b551508ad33 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -444,7 +444,7 @@ def test_dump_with_low_rows_max(self): assert_equals(get_csv_record_values(response.body), range(12)) - @mock.patch('ckanext.datastore.controller.PAGINATE_BY', 5) + @mock.patch('ckanext.datastore.blueprint.PAGINATE_BY', 5) def test_dump_pagination(self): resource = factories.Resource() data = { @@ -460,7 +460,7 @@ def test_dump_pagination(self): range(12)) @helpers.change_config('ckan.datastore.search.rows_max', '7') - @mock.patch('ckanext.datastore.controller.PAGINATE_BY', 5) + @mock.patch('ckanext.datastore.blueprint.PAGINATE_BY', 5) def test_dump_pagination_csv_with_limit(self): resource = factories.Resource() data = { @@ -477,7 +477,7 @@ def test_dump_pagination_csv_with_limit(self): range(11)) @helpers.change_config('ckan.datastore.search.rows_max', '7') - @mock.patch('ckanext.datastore.controller.PAGINATE_BY', 6) + @mock.patch('ckanext.datastore.blueprint.PAGINATE_BY', 6) def test_dump_pagination_csv_with_limit_same_as_paginate(self): resource = factories.Resource() data = { @@ -494,7 +494,7 @@ def test_dump_pagination_csv_with_limit_same_as_paginate(self): range(6)) @helpers.change_config('ckan.datastore.search.rows_max', '6') - @mock.patch('ckanext.datastore.controller.PAGINATE_BY', 5) + @mock.patch('ckanext.datastore.blueprint.PAGINATE_BY', 5) def test_dump_pagination_with_rows_max(self): resource = factories.Resource() data = { @@ -510,7 +510,7 @@ def test_dump_pagination_with_rows_max(self): range(7)) @helpers.change_config('ckan.datastore.search.rows_max', '6') - @mock.patch('ckanext.datastore.controller.PAGINATE_BY', 6) + @mock.patch('ckanext.datastore.blueprint.PAGINATE_BY', 6) def test_dump_pagination_with_rows_max_same_as_paginate(self): resource = factories.Resource() data = { @@ -526,7 +526,7 @@ def test_dump_pagination_with_rows_max_same_as_paginate(self): range(7)) @helpers.change_config('ckan.datastore.search.rows_max', '7') - @mock.patch('ckanext.datastore.controller.PAGINATE_BY', 5) + @mock.patch('ckanext.datastore.blueprint.PAGINATE_BY', 5) def test_dump_pagination_json_with_limit(self): resource = factories.Resource() data = { @@ -543,7 +543,7 @@ def test_dump_pagination_json_with_limit(self): range(6)) @helpers.change_config('ckan.datastore.search.rows_max', '6') - @mock.patch('ckanext.datastore.controller.PAGINATE_BY', 5) + @mock.patch('ckanext.datastore.blueprint.PAGINATE_BY', 5) def test_dump_pagination_json_with_rows_max(self): resource = factories.Resource() data = { From 7587ca20d715242a8b7f50300b329416fdfd67bc Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 19 Sep 2019 14:44:50 +0300 Subject: [PATCH 8/9] rename datastore.view to datastore.blueprint --- ckan/cli/datastore.py | 2 +- ckanext/datastore/{view.py => blueprint.py} | 0 ckanext/datastore/commands.py | 2 +- ckanext/datastore/plugin.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename ckanext/datastore/{view.py => blueprint.py} (100%) diff --git a/ckan/cli/datastore.py b/ckan/cli/datastore.py index 4c948ab0b30..0320f924d75 100644 --- a/ckan/cli/datastore.py +++ b/ckan/cli/datastore.py @@ -11,7 +11,7 @@ import ckanext.datastore as datastore_module from ckanext.datastore.backend.postgres import identifier -from ckanext.datastore.view import DUMP_FORMATS, dump_to +from ckanext.datastore.blueprint import DUMP_FORMATS, dump_to log = logging.getLogger(__name__) diff --git a/ckanext/datastore/view.py b/ckanext/datastore/blueprint.py similarity index 100% rename from ckanext/datastore/view.py rename to ckanext/datastore/blueprint.py diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py index 2e8dcbecb81..2d27f30f52f 100644 --- a/ckanext/datastore/commands.py +++ b/ckanext/datastore/commands.py @@ -9,7 +9,7 @@ click_config_option, ) from ckanext.datastore.backend.postgres import identifier -from ckanext.datastore.view import DUMP_FORMATS, dump_to +from ckanext.datastore.blueprint import DUMP_FORMATS, dump_to import click diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index dd1ab9c88a2..ea2933779c6 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -19,7 +19,7 @@ DatastoreBackend ) from ckanext.datastore.backend.postgres import DatastorePostgresqlBackend -import ckanext.datastore.view as view +import ckanext.datastore.blueprint as view log = logging.getLogger(__name__) _get_or_bust = logic.get_or_bust From 5e6424faa8d221dc5eb9d4d74bba2f74c673032a Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 19 Sep 2019 15:15:46 +0300 Subject: [PATCH 9/9] remove iconfigurer.controller because blacklist for pep8 update --- ckanext/example_iconfigurer/controller.py | 24 ----------------------- 1 file changed, 24 deletions(-) delete mode 100644 ckanext/example_iconfigurer/controller.py diff --git a/ckanext/example_iconfigurer/controller.py b/ckanext/example_iconfigurer/controller.py deleted file mode 100644 index 8dc8720b3b7..00000000000 --- a/ckanext/example_iconfigurer/controller.py +++ /dev/null @@ -1,24 +0,0 @@ -# encoding: utf-8 - -import ckan.lib.base as base -import ckan.lib.helpers as helpers - -render = base.render - - -class MyExtController(base.BaseController): - - def config_one(self): - '''Render the config template with the first custom title.''' - - return render('admin/myext_config.html', - extra_vars={'title': 'My First Config Page'}) - - def config_two(self): - '''Render the config template with the second custom title.''' - return render('admin/myext_config.html', - extra_vars={'title': 'My Second Config Page'}) - - def build_extra_admin_nav(self): - '''Return results of helpers.build_extra_admin_nav for testing.''' - return helpers.build_extra_admin_nav()