Upgrade pycodestyle and requests

* Newer pycodestyle requires this newer requests version
  (See
https://stackoverflow.com/questions/44927351/importerror-cannot-import-name-dependencywarning
)
* A whole bunch of new PEP8 errors and warnings come with the new
pycodestyle version, requiring changes in the main code. Feels
good to iron out some of these oddities though.
* Bit unsure if the regex in ckan/cli/translation.py is right to start
with. Not sure this CLI command is used though.

ckan/cli/search_index.py

@@ -93,9 +93,9 @@ def start(ids):
     def chunks(l, n):
         u""" Yield n successive chunks from l."""
         newn = int(len(l) / n)
-        for i in range(0, n-1):
-            yield l[i*newn:i*newn+newn]
-        yield l[n*newn-newn:]
+        for i in range(0, n - 1):
+            yield l[i * newn:i * newn + newn]
+        yield l[n * newn - newn:]
 
     processes = []
     with flask_app.test_request_context():

ckan/cli/translation.py

@@ -33,7 +33,8 @@ def js():
     u'mangle', short_help=u'Mangle the zh_TW translations for testing.'
 )
 def mangle():
-    u'''This will mangle the zh_TW translations for translation coverage testing.
+    u'''This will mangle the zh_TW translations for translation coverage
+    testing.
 
     NOTE: This will destroy the current translations fot zh_TW
     '''
@@ -44,7 +45,7 @@ def mangle():
     # %(...)s  %s %0.3f %1$s %2$0.3f [1:...] {...} etc
 
     # sprintf bit after %
-    spf_reg_ex = u"\\+?(0|'.)?-?\\d*(.\\d*)?[\%bcdeufosxX]"
+    spf_reg_ex = u"\\+?(0|'.)?-?\\d*(.\\d*)?[\\%bcdeufosxX]"
 
     extract_reg_ex = u'(\\%\\([^\\)]*\\)' + spf_reg_ex + \
                      u'|\\[\\d*\\:[^\\]]*\\]' + \

ckan/config/install.py

@@ -11,7 +11,7 @@ class CKANInstaller(PylonsInstaller):
 
     def config_content(self, command, vars):
         ckan_version = ckan.__version__
-        ckan_base_version = re.sub('[^0-9\.]', '', ckan_version)
+        ckan_base_version = re.sub(r'[^0-9\.]', '', ckan_version)
         if ckan_base_version == ckan_version:
             ckan_doc_version = 'ckan-{0}'.format(ckan_version)
         else:

ckan/config/middleware/flask_app.py

@@ -134,7 +134,7 @@ def save_session(self, app, session, response):
             session_opts.get('session.type', 'file') == 'file'):
         cache_dir = app_conf.get('cache_dir') or app_conf.get('cache.dir')
         session_opts['session.data_dir'] = '{data_dir}/sessions'.format(
-                data_dir=cache_dir)
+            data_dir=cache_dir)
 
     app.wsgi_app = SessionMiddleware(app.wsgi_app, session_opts)
     app.session_interface = BeakerSessionInterface()
@@ -211,8 +211,8 @@ def hello_world_post():
                 'controller': controller,
                 'highlight_actions': action,
                 'needed': needed
-                }
             }
+        }
         config['routes.named_routes'].update(route)
 
     # Start other middleware

ckan/config/middleware/pylons_app.py

@@ -264,7 +264,7 @@ def execute_on_completion(application, config, callback):
     def inner(environ, start_response):
         try:
             result = application(environ, start_response)
-        except:
+        except Exception:
             callback(environ)
             raise
         # paste.fileapp converts non-file responses into list

ckan/controllers/api.py

@@ -274,7 +274,7 @@ def group_autocomplete(self):
         limit = request.params.get('limit', 20)
         try:
             limit = int(limit)
-        except:
+        except ValueError:
             limit = 20
         limit = min(50, limit)
 

ckan/controllers/group.py

@@ -806,7 +806,7 @@ def history(self, id):
                     revision_dict['timestamp'])
                 try:
                     dayHorizon = int(request.params.get('days'))
-                except:
+                except ValueError:
                     dayHorizon = 30
                 dayAge = (datetime.datetime.now() - revision_date).days
                 if dayAge >= dayHorizon:

ckan/controllers/package.py

@@ -230,7 +230,7 @@ def pager_url(q=None, page=None):
             # types any search page. Potential alternatives are do show them
             # on the default search page (dataset) or on one other search page
             search_all_type = config.get(
-                                  'ckan.search.show_all_types', 'dataset')
+                'ckan.search.show_all_types', 'dataset')
             search_all = False
 
             try:
@@ -258,7 +258,7 @@ def pager_url(q=None, page=None):
                 'tags': _('Tags'),
                 'res_format': _('Formats'),
                 'license_id': _('Licenses'),
-                }
+            }
 
             for facet in h.facets():
                 if facet in default_facet_titles:
@@ -471,7 +471,7 @@ def history(self, id):
                     revision_dict['timestamp'])
                 try:
                     dayHorizon = int(request.params.get('days'))
-                except:
+                except ValueError:
                     dayHorizon = 30
                 dayAge = (datetime.datetime.now() - revision_date).days
                 if dayAge >= dayHorizon:

ckan/controllers/util.py

@@ -31,7 +31,7 @@ def i18_js_strings(self, lang):
         ''' This is used to produce the translations for javascript. '''
         i18n.set_lang(lang)
         html = base.render('js_strings.html', cache_force=True)
-        html = re.sub('<[^\>]*>', '', html)
+        html = re.sub(r'<[^\>]*>', '', html)
         header = "text/javascript; charset=utf-8"
         base.response.headers['Content-type'] = header
         return html

ckan/i18n/check_po_files.py

@@ -20,7 +20,7 @@ def simple_conv_specs(s):
     See http://docs.python.org/library/stdtypes.html#string-formatting
 
     '''
-    simple_conv_specs_re = re.compile('\%\w')
+    simple_conv_specs_re = re.compile(r'\%\w')
     return simple_conv_specs_re.findall(s)
 
 
@@ -32,7 +32,7 @@ def mapping_keys(s):
     See http://docs.python.org/library/stdtypes.html#string-formatting
 
     '''
-    mapping_keys_re = re.compile('\%\([^\)]*\)\w')
+    mapping_keys_re = re.compile(r'\%\([^\)]*\)\w')
     return sorted(mapping_keys_re.findall(s))
 
 
@@ -44,7 +44,7 @@ def replacement_fields(s):
     See http://docs.python.org/library/string.html#formatstrings
 
     '''
-    repl_fields_re = re.compile('\{[^\}]*\}')
+    repl_fields_re = re.compile(r'\{[^\}]*\}')
     return sorted(repl_fields_re.findall(s))
 
 

ckan/lib/datapreview.py

@@ -259,7 +259,7 @@ def add_views_to_resource(context,
         if view_plugin.can_view({
             'resource': resource_dict,
             'package': dataset_dict
-                }):
+                }):  # noqa
             view = {'resource_id': resource_dict['id'],
                     'view_type': view_info['name'],
                     'title': view_info.get('default_title', _('View')),

ckan/lib/helpers.py

@@ -915,9 +915,9 @@ def map_pylons_to_flask_route_name(menu_item):
             LEGACY_ROUTE_NAMES.update(mappings)
 
     if menu_item in LEGACY_ROUTE_NAMES:
-        log.info('Route name "{}" is deprecated and will be removed.\
-                Please update calls to use "{}" instead'.format(
-                menu_item, LEGACY_ROUTE_NAMES[menu_item]))
+        log.info('Route name "{}" is deprecated and will be removed. '
+                 'Please update calls to use "{}" instead'
+                 .format(menu_item, LEGACY_ROUTE_NAMES[menu_item]))
     return LEGACY_ROUTE_NAMES.get(menu_item, menu_item)
 
 
@@ -1487,11 +1487,11 @@ def date_str_to_datetime(date_str):
            despite that not being part of the ISO format.
     '''
 
-    time_tuple = re.split('[^\d]+', date_str, maxsplit=5)
+    time_tuple = re.split(r'[^\d]+', date_str, maxsplit=5)
 
     # Extract seconds and microseconds
     if len(time_tuple) >= 6:
-        m = re.match('(?P<seconds>\d{2})(\.(?P<microseconds>\d{6}))?$',
+        m = re.match(r'(?P<seconds>\d{2})(\.(?P<microseconds>\d{6}))?$',
                      time_tuple[5])
         if not m:
             raise ValueError('Unable to parse %s as seconds.microseconds' %
@@ -2101,7 +2101,7 @@ def get_request_param(parameter_name, default=None):
 # but ignore trailing punctuation since it is probably not part of the link
 RE_MD_EXTERNAL_LINK = re.compile(
     r'(\bhttps?:\/\/[\w\-\.,@?^=%&;:\/~\\+#]*'
-    '[\w\-@?^=%&:\/~\\+#]'  # but last character can't be punctuation [.,;]
+    r'[\w\-@?^=%&:\/~\\+#]'  # but last character can't be punctuation [.,;]
     ')',
     flags=re.UNICODE
 )
@@ -2184,9 +2184,9 @@ def format_resource_items(items):
     blacklist = ['name', 'description', 'url', 'tracking_summary']
     output = []
     # regular expressions for detecting types in strings
-    reg_ex_datetime = '^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?$'
-    reg_ex_int = '^-?\d{1,}$'
-    reg_ex_float = '^-?\d{1,}\.\d{1,}$'
+    reg_ex_datetime = r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?$'
+    reg_ex_int = r'^-?\d{1,}$'
+    reg_ex_float = r'^-?\d{1,}\.\d{1,}$'
     for key, value in items:
         if not value or key in blacklist:
             continue

ckan/lib/mailer.py

@@ -126,7 +126,7 @@ def get_reset_link_body(user):
         'site_title': config.get('ckan.site_title'),
         'site_url': config.get('ckan.site_url'),
         'user_name': user.name,
-        }
+    }
     # NOTE: This template is translated
     return render_jinja2('emails/reset_password.txt', extra_vars)
 
@@ -141,7 +141,7 @@ def get_invite_body(user, group_dict=None, role=None):
         'site_title': config.get('ckan.site_title'),
         'site_url': config.get('ckan.site_url'),
         'user_name': user.name,
-        }
+    }
     if role:
         extra_vars['role_name'] = h.roles_translated().get(role, _(role))
     if group_dict:

ckan/lib/munge.py

@@ -56,10 +56,10 @@ def munge_title_to_name(name):
     # (make length less than max, in case we need a few for '_' chars
     # to de-clash names.)
     if len(name) > max_length:
-        year_match = re.match('.*?[_-]((?:\d{2,4}[-/])?\d{2,4})$', name)
+        year_match = re.match(r'.*?[_-]((?:\d{2,4}[-/])?\d{2,4})$', name)
         if year_match:
             year = year_match.groups()[0]
-            name = '%s-%s' % (name[:(max_length-len(year)-1)], year)
+            name = '%s-%s' % (name[:(max_length - len(year) - 1)], year)
         else:
             name = name[:max_length]
     name = _munge_to_length(name, model.PACKAGE_NAME_MIN_LENGTH,

ckan/logic/__init__.py

@@ -88,7 +88,7 @@ def summarise(error_dict):
             ''' Do some i18n stuff on the error_dict keys '''
 
             def prettify(field_name):
-                field_name = re.sub('(?<!\w)[Uu]rl(?!\w)', 'URL',
+                field_name = re.sub(r'(?<!\w)[Uu]rl(?!\w)', 'URL',
                                     field_name.replace('_', ' ').capitalize())
                 return _(field_name.replace('_', ' '))
 

ckan/logic/action/create.py

@@ -230,8 +230,8 @@ def package_create(context, data_dict):
         return pkg.id
 
     return _get_action('package_show')(
-                context.copy(), {'id': pkg.id}
-            )
+        context.copy(), {'id': pkg.id}
+    )
 
 
 def resource_create(context, data_dict):
@@ -392,7 +392,7 @@ def resource_view_create(context, data_dict):
 
     max_order = model.Session.query(
         func.max(model.ResourceView.order)
-        ).filter_by(resource_id=resource_id).first()
+    ).filter_by(resource_id=resource_id).first()
 
     order = 0
     if max_order[0] is not None:

ckan/logic/action/patch.py

@@ -38,7 +38,7 @@ def package_patch(context, data_dict):
         'session': context['session'],
         'user': context['user'],
         'auth_user_obj': context['auth_user_obj'],
-        }
+    }
 
     package_dict = _get_action('package_show')(
         show_context,
@@ -68,7 +68,7 @@ def resource_patch(context, data_dict):
         'session': context['session'],
         'user': context['user'],
         'auth_user_obj': context['auth_user_obj'],
-        }
+    }
 
     resource_dict = _get_action('resource_show')(
         show_context,
@@ -97,7 +97,7 @@ def group_patch(context, data_dict):
         'session': context['session'],
         'user': context['user'],
         'auth_user_obj': context['auth_user_obj'],
-        }
+    }
 
     group_dict = _get_action('group_show')(
         show_context,
@@ -127,7 +127,7 @@ def organization_patch(context, data_dict):
         'session': context['session'],
         'user': context['user'],
         'auth_user_obj': context['auth_user_obj'],
-        }
+    }
 
     organization_dict = _get_action('organization_show')(
         show_context,

ckan/migration/versions/081_set_datastore_active.py

@@ -45,8 +45,7 @@ def upgrade(migrate_engine):
                 ','.join(['\'{0}\''.format(_id[0])
                           for _id
                           in resources_in_datastore])
-                )
-            )
+            ))
             if resources.rowcount:
                 params = []
                 for resource in resources:

ckan/model/revision.py

@@ -15,8 +15,8 @@ def make_revisioned_table(base_table, frozen=False):
     @return revision table.
     '''
     base_table.append_column(
-            Column(u'revision_id', UnicodeText, ForeignKey(u'revision.id'))
-            )
+        Column(u'revision_id', UnicodeText, ForeignKey(u'revision.id'))
+    )
     newtable = Table(base_table.name + u'_revision', base_table.metadata)
     copy_table(base_table, newtable)
 
@@ -33,7 +33,7 @@ def make_revisioned_table(base_table, frozen=False):
     newtable.append_column(
         Column(u'continuity_id', pkcols[0].type,
                None if frozen else ForeignKey(fk_name))
-        )
+    )
 
     # TODO: why do we iterate all the way through rather than just using dict
     # functionality ...? Surely we always have a revision here ...

ckan/tests/controllers/test_api.py

@@ -265,7 +265,7 @@ def test_jsonp_works_on_get_requests(self):
             url=url,
             params={'callback': 'my_callback'},
         )
-        assert re.match('my_callback\(.*\);', res.body), res
+        assert re.match(r'my_callback\(.*\);', res.body), res
         # Unwrap JSONP callback (we want to look at the data).
         msg = res.body[len('my_callback') + 1:-2]
         res_dict = json.loads(msg)

ckan/tests/helpers.py

@@ -293,7 +293,7 @@ def webtest_submit(form, name=None, index=None, value=None, **args):
     '''
     fields = webtest_submit_fields(form, name, index=index, submit_value=value)
     if form.method.upper() != "GET":
-        args.setdefault("content_type",  form.enctype)
+        args.setdefault("content_type", form.enctype)
     return form.response.goto(form.action, method=form.method,
                               params=fields, **args)
 

ckan/tests/legacy/test_coding_standards.py

@@ -643,12 +643,18 @@ def test_pep8_pass(self):
     def find_pep8_errors(cls, filename=None, lines=None):
         try:
             sys.stdout = cStringIO.StringIO()
-            config = {}
+            config = {'ignore': [
+                # W503/W504 - breaking before/after binary operators is agreed
+                # to not be a concern and was changed to be ignored by default.
+                # However we overwrite the ignore list here, so add it back in.
+                # See: https://github.com/PyCQA/pycodestyle/issues/498
+                'W503', 'W504',
+            ]}
 
             # Ignore long lines on test files, as the test names can get long
             # when following our test naming standards.
             if cls._is_test(filename):
-                config['ignore'] = ['E501']
+                config['ignore'].append('E501')
 
             checker = pycodestyle.Checker(filename=filename, lines=lines,
                                           **config)
@@ -668,7 +674,7 @@ def find_pep8_errors(cls, filename=None, lines=None):
 
     @classmethod
     def _is_test(cls, filename):
-        return bool(re.search('(^|\W)test_.*\.py$', filename, re.IGNORECASE))
+        return bool(re.search(r'(^|\W)test_.*\.py$', filename, re.IGNORECASE))
 
 
 class TestActionAuth(object):

ckan/tests/logic/action/test_create.py

@@ -1046,7 +1046,7 @@ def test_create_group_return_id(self):
         )
 
         assert isinstance(group, str)
-        assert re.match('([a-f\d]{8}(-[a-f\d]{4}){3}-[a-f\d]{12}?)', group)
+        assert re.match(r'([a-f\d]{8}(-[a-f\d]{4}){3}-[a-f\d]{12}?)', group)
 
     def test_create_matches_show(self):
         user = factories.User()
@@ -1124,7 +1124,7 @@ def test_create_organization_return_id(self):
         )
 
         assert isinstance(org, str)
-        assert re.match('([a-f\d]{8}(-[a-f\d]{4}){3}-[a-f\d]{12}?)', org)
+        assert re.match(r'([a-f\d]{8}(-[a-f\d]{4}){3}-[a-f\d]{12}?)', org)
 
     def test_create_matches_show(self):
         user = factories.User()