From d65b63b1e553e5074984e05db262deb97e1845eb Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 12 Apr 2019 12:58:38 +0100 Subject: [PATCH] Upgrade pycodestyle and requests * Newer pycodestyle requires this newer requests version (See https://stackoverflow.com/questions/44927351/importerror-cannot-import-name-dependencywarning ) * A whole bunch of new PEP8 errors and warnings come with the new pycodestyle version, requiring changes in the main code. Feels good to iron out some of these oddities though. * Bit unsure if the regex in ckan/cli/translation.py is right to start with. Not sure this CLI command is used though. --- ckan/cli/search_index.py | 6 +++--- ckan/cli/translation.py | 5 +++-- ckan/config/install.py | 2 +- ckan/config/middleware/flask_app.py | 4 ++-- ckan/config/middleware/pylons_app.py | 2 +- ckan/controllers/api.py | 2 +- ckan/controllers/group.py | 2 +- ckan/controllers/package.py | 6 +++--- ckan/controllers/util.py | 2 +- ckan/i18n/check_po_files.py | 6 +++--- ckan/lib/datapreview.py | 2 +- ckan/lib/helpers.py | 18 +++++++++--------- ckan/lib/mailer.py | 4 ++-- ckan/lib/munge.py | 4 ++-- ckan/logic/__init__.py | 2 +- ckan/logic/action/create.py | 6 +++--- ckan/logic/action/patch.py | 8 ++++---- .../versions/081_set_datastore_active.py | 3 +-- ckan/model/revision.py | 6 +++--- ckan/tests/controllers/test_api.py | 2 +- ckan/tests/helpers.py | 2 +- ckan/tests/legacy/test_coding_standards.py | 12 +++++++++--- ckan/tests/logic/action/test_create.py | 4 ++-- ckan/tests/logic/action/test_get.py | 4 ++-- ckan/views/group.py | 4 ++-- ckan/views/home.py | 6 +++--- ckanext/datastore/backend/postgres.py | 9 +++++---- ckanext/datastore/controller.py | 6 +++--- ckanext/datatablesview/controller.py | 2 +- ckanext/example_idatasetform/plugin_v4.py | 2 +- .../tests/test_example_idatasetform.py | 4 ++-- ckanext/example_ivalidators/plugin.py | 2 +- ckanext/imageview/plugin.py | 4 ++-- ckanext/resourceproxy/controller.py | 2 +- dev-requirements.txt | 2 +- requirements.in | 2 +- requirements.txt | 2 +- 37 files changed, 84 insertions(+), 77 deletions(-) diff --git a/ckan/cli/search_index.py b/ckan/cli/search_index.py index c8e809cbd30..bc23b38c9c8 100644 --- a/ckan/cli/search_index.py +++ b/ckan/cli/search_index.py @@ -93,9 +93,9 @@ def start(ids): def chunks(l, n): u""" Yield n successive chunks from l.""" newn = int(len(l) / n) - for i in range(0, n-1): - yield l[i*newn:i*newn+newn] - yield l[n*newn-newn:] + for i in range(0, n - 1): + yield l[i * newn:i * newn + newn] + yield l[n * newn - newn:] processes = [] with flask_app.test_request_context(): diff --git a/ckan/cli/translation.py b/ckan/cli/translation.py index e138dd76404..b720192f60e 100644 --- a/ckan/cli/translation.py +++ b/ckan/cli/translation.py @@ -33,7 +33,8 @@ def js(): u'mangle', short_help=u'Mangle the zh_TW translations for testing.' ) def mangle(): - u'''This will mangle the zh_TW translations for translation coverage testing. + u'''This will mangle the zh_TW translations for translation coverage + testing. NOTE: This will destroy the current translations fot zh_TW ''' @@ -44,7 +45,7 @@ def mangle(): # %(...)s %s %0.3f %1$s %2$0.3f [1:...] {...} etc # sprintf bit after % - spf_reg_ex = u"\\+?(0|'.)?-?\\d*(.\\d*)?[\%bcdeufosxX]" + spf_reg_ex = u"\\+?(0|'.)?-?\\d*(.\\d*)?[\\%bcdeufosxX]" extract_reg_ex = u'(\\%\\([^\\)]*\\)' + spf_reg_ex + \ u'|\\[\\d*\\:[^\\]]*\\]' + \ diff --git a/ckan/config/install.py b/ckan/config/install.py index d6cb33c9fe6..e429360203e 100644 --- a/ckan/config/install.py +++ b/ckan/config/install.py @@ -11,7 +11,7 @@ class CKANInstaller(PylonsInstaller): def config_content(self, command, vars): ckan_version = ckan.__version__ - ckan_base_version = re.sub('[^0-9\.]', '', ckan_version) + ckan_base_version = re.sub(r'[^0-9\.]', '', ckan_version) if ckan_base_version == ckan_version: ckan_doc_version = 'ckan-{0}'.format(ckan_version) else: diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py index a8406bfd457..c6d437a7221 100644 --- a/ckan/config/middleware/flask_app.py +++ b/ckan/config/middleware/flask_app.py @@ -134,7 +134,7 @@ def save_session(self, app, session, response): session_opts.get('session.type', 'file') == 'file'): cache_dir = app_conf.get('cache_dir') or app_conf.get('cache.dir') session_opts['session.data_dir'] = '{data_dir}/sessions'.format( - data_dir=cache_dir) + data_dir=cache_dir) app.wsgi_app = SessionMiddleware(app.wsgi_app, session_opts) app.session_interface = BeakerSessionInterface() @@ -211,8 +211,8 @@ def hello_world_post(): 'controller': controller, 'highlight_actions': action, 'needed': needed - } } + } config['routes.named_routes'].update(route) # Start other middleware diff --git a/ckan/config/middleware/pylons_app.py b/ckan/config/middleware/pylons_app.py index c9fb2ed8f9a..a935b36feba 100644 --- a/ckan/config/middleware/pylons_app.py +++ b/ckan/config/middleware/pylons_app.py @@ -264,7 +264,7 @@ def execute_on_completion(application, config, callback): def inner(environ, start_response): try: result = application(environ, start_response) - except: + except Exception: callback(environ) raise # paste.fileapp converts non-file responses into list diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index 826a9b6d279..e5b4bddcdb9 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -274,7 +274,7 @@ def group_autocomplete(self): limit = request.params.get('limit', 20) try: limit = int(limit) - except: + except ValueError: limit = 20 limit = min(50, limit) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 4a0d774b312..46c5c2b1813 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -806,7 +806,7 @@ def history(self, id): revision_dict['timestamp']) try: dayHorizon = int(request.params.get('days')) - except: + except ValueError: dayHorizon = 30 dayAge = (datetime.datetime.now() - revision_date).days if dayAge >= dayHorizon: diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index a385ecb901c..ec9d12b0499 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -230,7 +230,7 @@ def pager_url(q=None, page=None): # types any search page. Potential alternatives are do show them # on the default search page (dataset) or on one other search page search_all_type = config.get( - 'ckan.search.show_all_types', 'dataset') + 'ckan.search.show_all_types', 'dataset') search_all = False try: @@ -258,7 +258,7 @@ def pager_url(q=None, page=None): 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), - } + } for facet in h.facets(): if facet in default_facet_titles: @@ -471,7 +471,7 @@ def history(self, id): revision_dict['timestamp']) try: dayHorizon = int(request.params.get('days')) - except: + except ValueError: dayHorizon = 30 dayAge = (datetime.datetime.now() - revision_date).days if dayAge >= dayHorizon: diff --git a/ckan/controllers/util.py b/ckan/controllers/util.py index 8242222e97e..0f84ebe933f 100644 --- a/ckan/controllers/util.py +++ b/ckan/controllers/util.py @@ -31,7 +31,7 @@ def i18_js_strings(self, lang): ''' This is used to produce the translations for javascript. ''' i18n.set_lang(lang) html = base.render('js_strings.html', cache_force=True) - html = re.sub('<[^\>]*>', '', html) + html = re.sub(r'<[^\>]*>', '', html) header = "text/javascript; charset=utf-8" base.response.headers['Content-type'] = header return html diff --git a/ckan/i18n/check_po_files.py b/ckan/i18n/check_po_files.py index ea0d58bba81..a3acb09ef5f 100755 --- a/ckan/i18n/check_po_files.py +++ b/ckan/i18n/check_po_files.py @@ -20,7 +20,7 @@ def simple_conv_specs(s): See http://docs.python.org/library/stdtypes.html#string-formatting ''' - simple_conv_specs_re = re.compile('\%\w') + simple_conv_specs_re = re.compile(r'\%\w') return simple_conv_specs_re.findall(s) @@ -32,7 +32,7 @@ def mapping_keys(s): See http://docs.python.org/library/stdtypes.html#string-formatting ''' - mapping_keys_re = re.compile('\%\([^\)]*\)\w') + mapping_keys_re = re.compile(r'\%\([^\)]*\)\w') return sorted(mapping_keys_re.findall(s)) @@ -44,7 +44,7 @@ def replacement_fields(s): See http://docs.python.org/library/string.html#formatstrings ''' - repl_fields_re = re.compile('\{[^\}]*\}') + repl_fields_re = re.compile(r'\{[^\}]*\}') return sorted(repl_fields_re.findall(s)) diff --git a/ckan/lib/datapreview.py b/ckan/lib/datapreview.py index cb995de2437..e6e336098b0 100644 --- a/ckan/lib/datapreview.py +++ b/ckan/lib/datapreview.py @@ -259,7 +259,7 @@ def add_views_to_resource(context, if view_plugin.can_view({ 'resource': resource_dict, 'package': dataset_dict - }): + }): # noqa view = {'resource_id': resource_dict['id'], 'view_type': view_info['name'], 'title': view_info.get('default_title', _('View')), diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index ccc2a08ef35..a5bfadee831 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -915,9 +915,9 @@ def map_pylons_to_flask_route_name(menu_item): LEGACY_ROUTE_NAMES.update(mappings) if menu_item in LEGACY_ROUTE_NAMES: - log.info('Route name "{}" is deprecated and will be removed.\ - Please update calls to use "{}" instead'.format( - menu_item, LEGACY_ROUTE_NAMES[menu_item])) + log.info('Route name "{}" is deprecated and will be removed. ' + 'Please update calls to use "{}" instead' + .format(menu_item, LEGACY_ROUTE_NAMES[menu_item])) return LEGACY_ROUTE_NAMES.get(menu_item, menu_item) @@ -1487,11 +1487,11 @@ def date_str_to_datetime(date_str): despite that not being part of the ISO format. ''' - time_tuple = re.split('[^\d]+', date_str, maxsplit=5) + time_tuple = re.split(r'[^\d]+', date_str, maxsplit=5) # Extract seconds and microseconds if len(time_tuple) >= 6: - m = re.match('(?P\d{2})(\.(?P\d{6}))?$', + m = re.match(r'(?P\d{2})(\.(?P\d{6}))?$', time_tuple[5]) if not m: raise ValueError('Unable to parse %s as seconds.microseconds' % @@ -2101,7 +2101,7 @@ def get_request_param(parameter_name, default=None): # but ignore trailing punctuation since it is probably not part of the link RE_MD_EXTERNAL_LINK = re.compile( r'(\bhttps?:\/\/[\w\-\.,@?^=%&;:\/~\\+#]*' - '[\w\-@?^=%&:\/~\\+#]' # but last character can't be punctuation [.,;] + r'[\w\-@?^=%&:\/~\\+#]' # but last character can't be punctuation [.,;] ')', flags=re.UNICODE ) @@ -2184,9 +2184,9 @@ def format_resource_items(items): blacklist = ['name', 'description', 'url', 'tracking_summary'] output = [] # regular expressions for detecting types in strings - reg_ex_datetime = '^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?$' - reg_ex_int = '^-?\d{1,}$' - reg_ex_float = '^-?\d{1,}\.\d{1,}$' + reg_ex_datetime = r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?$' + reg_ex_int = r'^-?\d{1,}$' + reg_ex_float = r'^-?\d{1,}\.\d{1,}$' for key, value in items: if not value or key in blacklist: continue diff --git a/ckan/lib/mailer.py b/ckan/lib/mailer.py index 00c111bf888..9069c44415f 100644 --- a/ckan/lib/mailer.py +++ b/ckan/lib/mailer.py @@ -126,7 +126,7 @@ def get_reset_link_body(user): 'site_title': config.get('ckan.site_title'), 'site_url': config.get('ckan.site_url'), 'user_name': user.name, - } + } # NOTE: This template is translated return render_jinja2('emails/reset_password.txt', extra_vars) @@ -141,7 +141,7 @@ def get_invite_body(user, group_dict=None, role=None): 'site_title': config.get('ckan.site_title'), 'site_url': config.get('ckan.site_url'), 'user_name': user.name, - } + } if role: extra_vars['role_name'] = h.roles_translated().get(role, _(role)) if group_dict: diff --git a/ckan/lib/munge.py b/ckan/lib/munge.py index 3ef97ba1c16..8db3438a83f 100644 --- a/ckan/lib/munge.py +++ b/ckan/lib/munge.py @@ -56,10 +56,10 @@ def munge_title_to_name(name): # (make length less than max, in case we need a few for '_' chars # to de-clash names.) if len(name) > max_length: - year_match = re.match('.*?[_-]((?:\d{2,4}[-/])?\d{2,4})$', name) + year_match = re.match(r'.*?[_-]((?:\d{2,4}[-/])?\d{2,4})$', name) if year_match: year = year_match.groups()[0] - name = '%s-%s' % (name[:(max_length-len(year)-1)], year) + name = '%s-%s' % (name[:(max_length - len(year) - 1)], year) else: name = name[:max_length] name = _munge_to_length(name, model.PACKAGE_NAME_MIN_LENGTH, diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index 815a86b3147..c5b3b90b4a9 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -88,7 +88,7 @@ def summarise(error_dict): ''' Do some i18n stuff on the error_dict keys ''' def prettify(field_name): - field_name = re.sub('(? 0 @@ -2037,8 +2037,9 @@ def create_function(name, arguments, rettype, definition, or_replace): _write_engine_execute(sql) except ProgrammingError as pe: already_exists = ( - u'function "{}" already exists with same argument types'.format(name) - in pe.args[0]) + u'function "{}" already exists with same argument types' + .format(name) + in pe.args[0]) key = u'name' if already_exists else u'definition' raise ValidationError({key: [_programming_error_summary(pe)]}) diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index 6455a301339..022490b891c 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -87,7 +87,7 @@ def dump(self, resource_id): 'plain', 'language', 'fields']}, - ) + ) except ObjectNotFound: abort(404, _('DataStore resource not found')) @@ -123,7 +123,7 @@ def dictionary(self, id, resource_id): 'id': f['id'], 'type': f['type'], 'info': fi if isinstance(fi, dict) else {} - } for f, fi in izip_longest(fields, info)]}) + } for f, fi in izip_longest(fields, info)]}) h.flash_success(_('Data Dictionary saved. Any type overrides will ' 'take effect when the resource is next uploaded ' @@ -172,7 +172,7 @@ def result_page(offs, lim): 'sort': sort, 'records_format': records_format, 'include_total': False, - }, **search_params)) + }, **search_params)) result = result_page(offset, limit) diff --git a/ckanext/datatablesview/controller.py b/ckanext/datatablesview/controller.py index 06d7e40d721..f45984fa499 100644 --- a/ckanext/datatablesview/controller.py +++ b/ckanext/datatablesview/controller.py @@ -105,7 +105,7 @@ def filtered_download(self, resource_view_id): u'filters': json.dumps(filters), u'format': request.params['format'], u'fields': u','.join(cols), - })) + })) def merge_filters(view_filters, user_filters_str): diff --git a/ckanext/example_idatasetform/plugin_v4.py b/ckanext/example_idatasetform/plugin_v4.py index 0b89e148398..a3d2948b048 100644 --- a/ckanext/example_idatasetform/plugin_v4.py +++ b/ckanext/example_idatasetform/plugin_v4.py @@ -61,7 +61,7 @@ def show_package_schema(self): 'country_code': [ tk.get_converter('convert_from_tags')('country_codes'), tk.get_validator('ignore_missing')] - }) + }) return schema def create_package_schema(self): diff --git a/ckanext/example_idatasetform/tests/test_example_idatasetform.py b/ckanext/example_idatasetform/tests/test_example_idatasetform.py index c19c2ca92c2..45038a25506 100644 --- a/ckanext/example_idatasetform/tests/test_example_idatasetform.py +++ b/ckanext/example_idatasetform/tests/test_example_idatasetform.py @@ -208,9 +208,9 @@ def test_package_show(self): class TestCustomSearch(object): - # @classmethod + # @classmethod # def _apply_config_changes(cls, cfg): - # cfg['ckan.plugins'] = 'example_idatasetform' + # cfg['ckan.plugins'] = 'example_idatasetform' @classmethod def setup_class(cls): diff --git a/ckanext/example_ivalidators/plugin.py b/ckanext/example_ivalidators/plugin.py index 26733148bd6..2fe8e4e2b62 100644 --- a/ckanext/example_ivalidators/plugin.py +++ b/ckanext/example_ivalidators/plugin.py @@ -14,7 +14,7 @@ def get_validators(self): u'equals_fortytwo': equals_fortytwo, u'negate': negate, u'unicode_only': unicode_please, - } + } def equals_fortytwo(value): diff --git a/ckanext/imageview/plugin.py b/ckanext/imageview/plugin.py index 8e748641cf1..13a8758fc48 100644 --- a/ckanext/imageview/plugin.py +++ b/ckanext/imageview/plugin.py @@ -19,8 +19,8 @@ class ImageView(p.SingletonPlugin): def update_config(self, config): p.toolkit.add_template_directory(config, 'theme/templates') self.formats = config.get( - 'ckan.preview.image_formats', - DEFAULT_IMAGE_FORMATS).split() + 'ckan.preview.image_formats', + DEFAULT_IMAGE_FORMATS).split() def info(self): return {'name': 'image_view', diff --git a/ckanext/resourceproxy/controller.py b/ckanext/resourceproxy/controller.py index d1c8a0129b5..40371d9c90f 100644 --- a/ckanext/resourceproxy/controller.py +++ b/ckanext/resourceproxy/controller.py @@ -30,7 +30,7 @@ def proxy_resource(context, data_dict): resource = logic.get_action('resource_show')(context, {'id': resource_id}) except logic.NotFound: - base.abort(404, _('Resource not found')) + base.abort(404, _('Resource not found')) url = resource['url'] parts = urlparse.urlsplit(url) diff --git a/dev-requirements.txt b/dev-requirements.txt index f231be1ee67..99e95b0737c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -7,7 +7,7 @@ factory-boy==2.1.1 Flask-DebugToolbar==0.10.1 httpretty==0.8.3 mock==2.0.0 -pycodestyle==2.2.0 +pycodestyle==2.5.0 pip-tools==2.0.2 pyfakefs==3.2 Sphinx==1.7.5 diff --git a/requirements.in b/requirements.in index 0b0bf21699e..1b019936f9f 100644 --- a/requirements.in +++ b/requirements.in @@ -22,7 +22,7 @@ pytz==2016.7 pyutilib.component.core==4.6.4 repoze.who-friendlyform==1.0.8 repoze.who==2.3 -requests==2.20.0 +requests==2.21.0 Routes==1.13 rq==0.6.0 simplejson==3.10.0 diff --git a/requirements.txt b/requirements.txt index 822749a8769..1ecca3386c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,7 +45,7 @@ redis==2.10.6 # via rq repoze.lru==0.7 # via routes repoze.who-friendlyform==1.0.8 repoze.who==2.3 -requests==2.20.0 +requests==2.21.0 routes==1.13 rq==0.6.0 simplejson==3.10.0