fixed make_valid_url encoding bug

goodtables/utilities/helpers.py

@@ -216,13 +216,18 @@ def make_valid_url(url):
         return (glue).join(quoted)
 
     scheme, netloc, path, query, fragment = compat.urlsplit(url)
-    encoded_path = url_encode_non_ascii(path)
-    encoded_query = url_encode_non_ascii(query)
-    new_url_tuple = (scheme, netloc, encoded_path, encoded_query, fragment)
+    path = url_encode_non_ascii(path)
+    query = url_encode_non_ascii(query)
+    new_url_tuple = (scheme, netloc, path, query, fragment)
     quoted_url = compat.urlunsplit(new_url_tuple)
     return quoted_url
 
 
-def url_encode_non_ascii(text):
+def url_encode_non_ascii(element):
     # http://stackoverflow.com/questions/4389572/how-to-fetch-a-non-ascii-url-with-python-urlopen
-    return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), text)
+    pattern = '[\x80-\xFF]'.encode('utf-8')
+    element = element.encode('utf-8')
+    replace = lambda char: ('%%%02x' % ord(char.group(0))).encode('utf-8')
+    element = re.sub(pattern, replace, element)
+    element = element.decode('utf-8')
+    return element

tests/test_utilities.py

@@ -32,6 +32,11 @@ def test_make_valid_url(self):
         assertion = '£' not in utilities.helpers.make_valid_url(url)
         self.assertTrue(assertion)
 
+    def test_make_valid_url_urlencode(self):
+        url = 'http://data.defra.gov.uk/ops/government_procurement_card/over_£500_GPC_apr_2013.csv'
+        valid_url = 'http://data.defra.gov.uk/ops/government_procurement_card/over_%c2%a3500_GPC_apr_2013.csv'
+        self.assertEqual(utilities.helpers.make_valid_url(url), valid_url)
+
     def test_make_valid_url_dont_break_query(self):
         url = 'http://next.openspending.org/fdp-adapter/convert?url=https%3A%2F%2Fraw.githubusercontent.com%2Fkravets-levko%2Fdata%2Fmaster%2Ftest.xlsx.csv'
         self.assertEqual(utilities.helpers.make_valid_url(url), url)