Merge branch 'master' of github.com:okfn/ckan

ckan/controllers/api.py

@@ -832,8 +832,11 @@ def make_unicode(entity):
                 raise ValueError(msg)
             cls.log.debug('Retrieved request body: %r' % request.body)
             if not request_data:
-                msg = "No request body data"
-                raise ValueError(msg)
+                if not try_url_params:
+                    msg = "No request body data"
+                    raise ValueError(msg)
+                else:
+                    request_data = {}
         if request_data:
             try:
                 request_data = h.json.loads(request_data, encoding='utf8')

ckan/lib/datapreview.py

@@ -22,10 +22,14 @@ def compare_domains(urls):
     for url in urls:
         # all urls are interpreted as absolute urls,
         # except for urls that start with a /
-        if not urlparse.urlparse(url).scheme and not url.startswith('/'):
-            url = '//' + url
-        parsed = urlparse.urlparse(url.lower(), 'http')
-        domain = (parsed.scheme, parsed.hostname, parsed.port)
+        try:
+            if not urlparse.urlparse(url).scheme and not url.startswith('/'):
+                url = '//' + url
+            parsed = urlparse.urlparse(url.lower(), 'http')
+            domain = (parsed.scheme, parsed.hostname, parsed.port)
+        except ValueError:
+            # URL is so messed up that even urlparse can't stand it
+            return False
 
         if not first_domain:
             first_domain = domain

ckan/lib/helpers.py

@@ -548,15 +548,28 @@ def _search_url(params):
     return _url_with_params(url, params)
 
 
-def sorted_extras(list_):
-    ''' Used for outputting package extras '''
+def sorted_extras(package_extras, auto_clean=False, subs=None):
+    ''' Used for outputting package extras
+
+    :param package_extras: the package extras
+    :type package_extras: dict
+    :param auto_clean: If true capitalize and replace -_ with spaces
+    :type auto_clean: bool
+    :param subs: substitutes to use instead of given keys
+    :type subs: dict {'key': 'replacement'}
+    '''
+
     output = []
-    for extra in sorted(list_, key=lambda x: x['key']):
+    for extra in sorted(package_extras, key=lambda x: x['key']):
         if extra.get('state') == 'deleted':
             continue
         k, v = extra['key'], extra['value']
         if k in g.package_hide_extras:
             continue
+        if subs and k in subs:
+            k = subs[k]
+        elif auto_clean:
+            k = k.replace('_', ' ').replace('-', ' ').title()
         if isinstance(v, (list, tuple)):
             v = ", ".join(map(unicode, v))
         output.append((k, v))

ckan/migration/versions/067_turn_extras_to_strings.py

@@ -7,7 +7,7 @@ def upgrade(migrate_engine):
         revision_tables = 'package_extra_revision group_extra_revision'
 
         for table in tables.split():
-            sql = """select id, value from {table} where left(value,1) = '"' """.format(table=table)
+            sql = """select id, value from {table} where substr(value,0,1) = '"' """.format(table=table)
             results = connection.execute(sql)
             for result in results:
                 id, value = result
@@ -16,7 +16,7 @@ def upgrade(migrate_engine):
                                    json.loads(value), id)
 
         for table in revision_tables.split():
-            sql = """select id, revision_id, value from {table} where left(value,1) = '"' """.format(table=table)
+            sql = """select id, revision_id, value from {table} where substr(value,0,1) = '"' """.format(table=table)
 
             results = connection.execute(sql)
             for result in results:

ckan/tests/lib/test_datapreview.py

@@ -1,20 +1,28 @@
+# -*- coding: utf-8 -*-
 import ckan.lib.datapreview as datapreview
 
+
 class TestDataPreview():
     def test_compare_domains(self):
         ''' see https://en.wikipedia.org/wiki/Same_origin_policy
         '''
         comp = datapreview.compare_domains
-        assert comp(['http://www.okfn.org', 'http://www.okfn.org']) == True
-        assert comp(['http://www.okfn.org', 'http://www.okfn.org', 'http://www.okfn.org']) == True
-        assert comp(['http://www.OKFN.org', 'http://www.okfn.org', 'http://www.okfn.org/test/foo.html']) == True
-        assert comp(['http://okfn.org', 'http://okfn.org']) == True
-        assert comp(['www.okfn.org', 'http://www.okfn.org']) == True
-        assert comp(['//www.okfn.org', 'http://www.okfn.org']) == True
+        assert comp(['http://www.okfn.org', 'http://www.okfn.org']) is True
+        assert comp(['http://www.okfn.org', 'http://www.okfn.org', 'http://www.okfn.org']) is True
+        assert comp(['http://www.OKFN.org', 'http://www.okfn.org', 'http://www.okfn.org/test/foo.html']) is True
+        assert comp(['http://okfn.org', 'http://okfn.org']) is True
+        assert comp(['www.okfn.org', 'http://www.okfn.org']) is True
+        assert comp(['//www.okfn.org', 'http://www.okfn.org']) is True
+
+        assert comp(['http://www.okfn.org', 'https://www.okfn.org']) is False
+        assert comp(['http://www.okfn.org:80', 'http://www.okfn.org:81']) is False
+        assert comp(['http://www.okfn.org', 'http://www.okfn.de']) is False
+        assert comp(['http://de.okfn.org', 'http://www.okfn.org']) is False
+
+        assert comp(['http://de.okfn.org', 'http:www.foo.com']) is False
 
-        assert comp(['http://www.okfn.org', 'https://www.okfn.org']) == False
-        assert comp(['http://www.okfn.org:80', 'http://www.okfn.org:81']) == False
-        assert comp(['http://www.okfn.org', 'http://www.okfn.de']) == False
-        assert comp(['http://de.okfn.org', 'http://www.okfn.org']) == False
+        assert comp(['httpö://wöwöwö.ckan.dö', 'www.ckän.örg']) is False
+        assert comp(['www.ckän.örg', 'www.ckän.örg']) is True
 
-        assert comp(['http://de.okfn.org', 'http:www.foo.com']) == False
+        # Wrong URL. Makes urlparse choke
+        assert comp(['http://Server=cda3; Service=sde:sqlserver:cda3; Database=NationalDatasets; User=sde; Version=sde.DEFAULT', 'http://www.okf.org']) is False

ckan/tests/logic/test_action.py

@@ -1,5 +1,6 @@
 import re
 import json
+import urllib
 from pprint import pprint
 from nose.tools import assert_equal, assert_raises
 from nose.plugins.skip import SkipTest
@@ -66,6 +67,12 @@ def test_01_package_list(self):
         assert res['help'].startswith(
             "Return a list of the names of the site's datasets (packages).")
 
+        # Test GET request
+        res = json.loads(self.app.get('/api/action/package_list').body)
+        assert len(res['result']) == 2
+        assert 'warandpeace' in res['result']
+        assert 'annakarenina' in res['result']
+
     def test_01_package_show(self):
         anna_id = model.Package.by_name(u'annakarenina').id
         postparams = '%s=1' % json.dumps({'id': anna_id})
@@ -453,6 +460,11 @@ def test_13_group_list(self):
         assert res_obj['help'].startswith(
                 "Return a list of the names of the site's groups.")
 
+        # Test GET request
+        res = self.app.get('/api/action/group_list')
+        res_obj = json.loads(res.body)
+        assert res_obj['result'] == ['david', 'roger']
+
         #Get all fields
         postparams = '%s=1' % json.dumps({'all_fields':True})
         res = self.app.post('/api/action/group_list', params=postparams)
@@ -1176,19 +1188,46 @@ def teardown_class(self):
         model.repo.rebuild_db()
 
     def test_1_basic(self):
-        postparams = '%s=1' % json.dumps({
+        params = {
                 'q':'tolstoy',
                 'facet.field': ('groups', 'tags', 'res_format', 'license'),
                 'rows': 20,
                 'start': 0,
-            })
+            }
+        postparams = '%s=1' % json.dumps(params)
         res = self.app.post('/api/action/package_search', params=postparams)
         res = json.loads(res.body)
         result = res['result']
         assert_equal(res['success'], True)
         assert_equal(result['count'], 1)
         assert_equal(result['results'][0]['name'], 'annakarenina')
 
+        # Test GET request
+        url_params = urllib.urlencode(params)
+        res = self.app.get('/api/action/package_search?{0}'.format(url_params))
+        res = json.loads(res.body)
+        result = res['result']
+        assert_equal(res['success'], True)
+        assert_equal(result['count'], 1)
+        assert_equal(result['results'][0]['name'], 'annakarenina')
+
+    def test_1_basic_no_params(self):
+        postparams = '%s=1' % json.dumps({})
+        res = self.app.post('/api/action/package_search', params=postparams)
+        res = json.loads(res.body)
+        result = res['result']
+        assert_equal(res['success'], True)
+        assert_equal(result['count'], 2)
+        assert_equal(result['results'][0]['name'], 'annakarenina')
+
+        # Test GET request
+        res = self.app.get('/api/action/package_search')
+        res = json.loads(res.body)
+        result = res['result']
+        assert_equal(res['success'], True)
+        assert_equal(result['count'], 2)
+        assert_equal(result['results'][0]['name'], 'annakarenina')
+
     def test_2_bad_param(self):
         postparams = '%s=1' % json.dumps({
                 'sort':'metadata_modified',