From 7e2430a8fac88cf69bb33876f10c4556719f1f1a Mon Sep 17 00:00:00 2001
From: Sergey Motornyuk <sergey.motornyuk@linkdigital.com.au>
Date: Thu, 16 Mar 2017 16:33:40 +0200
Subject: [PATCH] Use json.dumps for nested fields in datastore_dump

---
 ckanext/datastore/controller.py      | 24 +++++++++++++++++-
 ckanext/datastore/tests/test_dump.py | 38 +++++++++++++++++++---------
 2 files changed, 49 insertions(+), 13 deletions(-)

diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py
index b872c1d3a78..8f1398ec8c7 100644
--- a/ckanext/datastore/controller.py
+++ b/ckanext/datastore/controller.py
@@ -1,5 +1,6 @@
 # encoding: utf-8
 
+import json
 import StringIO
 import unicodecsv as csv
 
@@ -12,6 +13,20 @@
 from ckan.common import request
 
 
+def _dump_nested(column, record):
+    name, ctype = column
+    value = record[name]
+
+    is_nested = (
+        ctype == 'json' or
+        ctype.startswith('_') or
+        ctype.endswith(']')
+    )
+    if is_nested:
+        return json.dumps(value)
+    return value
+
+
 class DatastoreController(base.BaseController):
     def dump(self, resource_id):
         context = {
@@ -41,6 +56,13 @@ def dump(self, resource_id):
         header = [x['id'] for x in result['fields']]
         wr.writerow(header)
 
+        columns = [
+            (x['id'], x['type'])
+            for x in result['fields']]
+
         for record in result['records']:
-            wr.writerow([record[column] for column in header])
+            wr.writerow([
+                _dump_nested(column, record)
+                for column in columns])
+
         return f.getvalue()
diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py
index b966bedf7fa..f488011d8fe 100644
--- a/ckanext/datastore/tests/test_dump.py
+++ b/ckanext/datastore/tests/test_dump.py
@@ -3,7 +3,7 @@
 import json
 
 import nose
-from nose.tools import assert_equals
+from nose.tools import assert_equals, assert_in
 from ckan.common import config
 import sqlalchemy.orm as orm
 import paste.fixture
@@ -39,14 +39,16 @@ def setup_class(cls):
             'fields': [{'id': u'b\xfck', 'type': 'text'},
                        {'id': 'author', 'type': 'text'},
                        {'id': 'published'},
-                       {'id': u'characters', u'type': u'_text'}],
+                       {'id': u'characters', u'type': u'_text'},
+                       {'id': 'random_letters', 'type': 'text[]'}],
             'records': [{u'b\xfck': 'annakarenina',
-                        'author': 'tolstoy',
-                        'published': '2005-03-01',
-                        'nested': ['b', {'moo': 'moo'}],
-                        u'characters': [u'Princess Anna', u'Sergius']},
+                         'author': 'tolstoy',
+                         'published': '2005-03-01',
+                         'nested': ['b', {'moo': 'moo'}],
+                         u'characters': [u'Princess Anna', u'Sergius'],
+                         'random_letters': ['a', 'e', 'x']},
                         {u'b\xfck': 'warandpeace', 'author': 'tolstoy',
-                         'nested': {'a': 'b'}}]
+                         'nested': {'a': 'b'}, 'random_letters': []}]
         }
         postparams = '%s=1' % json.dumps(cls.data)
         auth = {'Authorization': str(cls.sysadmin_user.apikey)}
@@ -69,10 +71,13 @@ def test_dump_basic(self):
         res = self.app.get('/datastore/dump/{0}'.format(str(
             self.data['resource_id'])), extra_environ=auth)
         content = res.body.decode('utf-8')
-        expected = u'_id,b\xfck,author,published,characters,nested'
+
+        expected = (
+            u'_id,b\xfck,author,published'
+            u',characters,random_letters,nested')
         assert_equals(content[:len(expected)], expected)
-        assert 'warandpeace' in content
-        assert "[u'Princess Anna', u'Sergius']" in content
+        assert_in('warandpeace', content)
+        assert_in('"[""Princess Anna"", ""Sergius""]"', content)
 
         # get with alias instead of id
         res = self.app.get('/datastore/dump/{0}'.format(str(
@@ -88,6 +93,15 @@ def test_dump_limit(self):
         res = self.app.get('/datastore/dump/{0}?limit=1'.format(str(
             self.data['resource_id'])), extra_environ=auth)
         content = res.body.decode('utf-8')
-        expected = u'_id,b\xfck,author,published,characters,nested'
+
+        expected = (u'_id,b\xfck,author,published'
+                    u',characters,random_letters,nested')
         assert_equals(content[:len(expected)], expected)
-        assert_equals(len(content), 148)
+
+        expected_content = (
+            u'_id,b\xfck,author,published,characters,random_letters,'
+            u'nested\r\n1,annakarenina,tolstoy,2005-03-01T00:00:00,'
+            u'"[""Princess Anna"", ""Sergius""]",'
+            u'"[""a"", ""e"", ""x""]","[""b"", '
+            u'{""moo"": ""moo""}]"\r\n')
+        assert_equals(content, expected_content)