From 641fc7f7aae561bdbe1748ef90fee0369fc7e63c Mon Sep 17 00:00:00 2001 From: Steve Lamb Date: Tue, 4 Feb 2014 11:34:09 -0500 Subject: [PATCH 1/4] Fix minor issue in csv assertion the comparison loop was using zip(), which will only pair values up to the length of the shorter sequence. So in this case, if your test data had 3 rows, but your output data only had 2, the test would still pass even though that's clearly not OK. Using izip_longest() with padding, all elements of the two sequences are paired together. Also factor out this method so it can be used by another class. --- test_app/djqscsv_tests/tests.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/test_app/djqscsv_tests/tests.py b/test_app/djqscsv_tests/tests.py index 5b30558..9b3ab84 100644 --- a/test_app/djqscsv_tests/tests.py +++ b/test_app/djqscsv_tests/tests.py @@ -4,6 +4,7 @@ from django import VERSION as DJANGO_VERSION import csv +import itertools from .context import djqscsv @@ -89,7 +90,21 @@ def test_generate_filename(self): r'person_export_[0-9]{8}.csv') -class WriteCSVDataTests(TestCase): +class CSVTestCase(TestCase): + + def assertMatchesCsv(self, csv_file, expected_data): + csv_data = csv.reader(csv_file) + iteration_happened = False + test_pairs = itertools.izip_longest(csv_data, expected_data, + fillvalue=[]) + for csv_row, expected_row in test_pairs: + iteration_happened = True + self.assertEqual(csv_row, expected_row) + + self.assertTrue(iteration_happened, "The CSV does not contain data.") + + +class WriteCSVDataTests(CSVTestCase): def setUp(self): self.qs = create_people_and_get_queryset() @@ -112,15 +127,6 @@ def setUp(self): ['vetch', 'iffish', 'wizard'], ['nemmerle', 'roke', 'arch mage']] - def assertMatchesCsv(self, csv_file, expected_data): - csv_data = csv.reader(csv_file) - iteration_happened = False - for csv_row, expected_row in zip(csv_data, expected_data): - iteration_happened = True - self.assertEqual(csv_row, expected_row) - - self.assertTrue(iteration_happened, "The CSV does not contain data.") - def test_write_csv_full_terse(self): obj = StringIO() djqscsv.write_csv(self.qs, obj, use_verbose_names=False) From 483c82ef6807a9b69c0d556b9003a5325d5df29b Mon Sep 17 00:00:00 2001 From: Steve Lamb Date: Tue, 4 Feb 2014 11:38:55 -0500 Subject: [PATCH 2/4] Improve test data It is useful to have more test data in the main dataset, so I moved Ged there. This will be necessary for doing more advanced ORM operations in tests of newer features. --- test_app/djqscsv_tests/tests.py | 20 ++++++++++++-------- test_app/djqscsv_tests/util.py | 4 +++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/test_app/djqscsv_tests/tests.py b/test_app/djqscsv_tests/tests.py index 9b3ab84..3ee7486 100644 --- a/test_app/djqscsv_tests/tests.py +++ b/test_app/djqscsv_tests/tests.py @@ -51,12 +51,12 @@ def test_non_csv_raises_2(self): class SanitizeUnicodeRecordTests(TestCase): def test_sanitize(self): - record = {'name': 'Ged', - 'nickname': u'\ufeffSparrowhawk'} + record = {'name': 'Tenar', + 'nickname': u'\ufeffThe White Lady of Gont'} sanitized = djqscsv._sanitize_unicode_record(record) self.assertEqual(sanitized, - {'name': 'Ged', - 'nickname': '\xef\xbb\xbfSparrowhawk'}) + {'name': 'Tenar', + 'nickname': '\xef\xbb\xbfThe White Lady of Gont'}) class AppendDatestampTests(TestCase): @@ -112,20 +112,24 @@ def setUp(self): self.full_verbose_csv = [ ['\xef\xbb\xbfID', 'Person\'s name', 'address', 'Info on Person'], ['1', 'vetch', 'iffish', 'wizard'], - ['2', 'nemmerle', 'roke', 'arch mage']] + ['2', 'nemmerle', 'roke', 'deceased arch mage'], + ['3', 'ged', 'gont', 'former arch mage']] self.full_csv = [['\xef\xbb\xbfid', 'name', 'address', 'info'], ['1', 'vetch', 'iffish', 'wizard'], - ['2', 'nemmerle', 'roke', 'arch mage']] + ['2', 'nemmerle', 'roke', 'deceased arch mage'], + ['3', 'ged', 'gont', 'former arch mage']] self.limited_verbose_csv = [ ['\xef\xbb\xbfPerson\'s name', 'address', 'Info on Person'], ['vetch', 'iffish', 'wizard'], - ['nemmerle', 'roke', 'arch mage']] + ['nemmerle', 'roke', 'deceased arch mage'], + ['ged', 'gont', 'former arch mage']] self.limited_csv = [['\xef\xbb\xbfname', 'address', 'info'], ['vetch', 'iffish', 'wizard'], - ['nemmerle', 'roke', 'arch mage']] + ['nemmerle', 'roke', 'deceased arch mage'], + ['ged', 'gont', 'former arch mage']] def test_write_csv_full_terse(self): obj = StringIO() diff --git a/test_app/djqscsv_tests/util.py b/test_app/djqscsv_tests/util.py index b68cf82..ad1fd2e 100644 --- a/test_app/djqscsv_tests/util.py +++ b/test_app/djqscsv_tests/util.py @@ -3,7 +3,9 @@ def create_people_and_get_queryset(): Person.objects.create(name='vetch', address='iffish', info='wizard') Person.objects.create(name='nemmerle', address='roke', - info='arch mage') + info='deceased arch mage') + Person.objects.create(name='ged', address='gont', + info='former arch mage') return Person.objects.all() From 4ba44957368bd64d1497a29bd2564efd7563db54 Mon Sep 17 00:00:00 2001 From: Steve Lamb Date: Tue, 4 Feb 2014 11:40:14 -0500 Subject: [PATCH 3/4] Fix bug in record sanitation One of the unit tests caught this - nontruthy values were being discarded on export. Yikes. --- djqscsv/djqscsv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/djqscsv/djqscsv.py b/djqscsv/djqscsv.py index 6751041..ff43795 100644 --- a/djqscsv/djqscsv.py +++ b/djqscsv/djqscsv.py @@ -137,7 +137,7 @@ def _sanitize_value(value): obj = {} for key, val in six.iteritems(record): - if val: + if val is not None: obj[_sanitize_value(key)] = _sanitize_value(val) return obj From fe6491112c38315978e3af8bc7ecf30354280cd1 Mon Sep 17 00:00:00 2001 From: Steve Lamb Date: Tue, 4 Feb 2014 11:44:22 -0500 Subject: [PATCH 4/4] Add support for custom ordering This addresses a bit of a pain point in the best way I can think of. Normally, you can specify column ordering in the .values() call to the Queryset. This works fine for normal fields, and for walking relationships, but unfortunately does not work for extra_select fields. When support was added for extra_select fields, they were just tacked on to the end of the regular ordering. We now provide a 'field_order' argument that takes a list of field names to use for ordering when including extra_select data. As the tests show, a complete ordering is not required. The default order is to honor the order provided by the Model class, or the .values() call with extra_select fields added at the end. Any fields put in the ordering list will be placed at the beginning, and everything else will be added at the end, honoring their previous ordering rule. --- djqscsv/djqscsv.py | 24 +++++++++++++++++------- setup.py | 2 +- test_app/djqscsv_tests/tests.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/djqscsv/djqscsv.py b/djqscsv/djqscsv.py index ff43795..3a685e6 100644 --- a/djqscsv/djqscsv.py +++ b/djqscsv/djqscsv.py @@ -23,7 +23,8 @@ class CSVException(Exception): def render_to_csv_response(queryset, filename=None, append_datestamp=False, - field_header_map=None, use_verbose_names=True): + field_header_map=None, use_verbose_names=True, + field_order=None): """ provides the boilerplate for making a CSV http response. takes a filename or generates one from the queryset's model. @@ -40,13 +41,13 @@ def render_to_csv_response(queryset, filename=None, append_datestamp=False, response['Content-Disposition'] = 'attachment; filename=%s;' % filename response['Cache-Control'] = 'no-cache' - write_csv(queryset, response, field_header_map, use_verbose_names) + write_csv(queryset, response, field_header_map, use_verbose_names, field_order) return response def write_csv(queryset, file_obj, field_header_map=None, - use_verbose_names=True): + use_verbose_names=True, field_order=None): """ The main worker function. Writes CSV data to a file object based on the contents of the queryset. @@ -64,16 +65,25 @@ def write_csv(queryset, file_obj, field_header_map=None, try: field_names = values_qs.field_names - extra_columns = list(values_qs.query.extra_select) - if extra_columns: - # TODO: provide actual ordering - field_names += extra_columns except AttributeError: # in django1.5, empty querysets trigger # this exception, but not django 1.6 raise CSVException("Empty queryset provided to exporter.") + extra_columns = list(values_qs.query.extra_select) + if extra_columns: + field_names += extra_columns + + if field_order: + # go through the field_names and put the ones + # that appear in the ordering list first + field_names = ([field for field in field_order + if field in field_names] + + [field for field in field_names + if field not in field_order]) + + writer = csv.DictWriter(file_obj, field_names) # verbose_name defaults to the raw field name, so in either case diff --git a/setup.py b/setup.py index 52c66ab..f81c657 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-queryset-csv', - version='0.2.2', + version='0.2.3', description='A simple python module for writing querysets to csv', long_description=open('README.md').read(), author=author, diff --git a/test_app/djqscsv_tests/tests.py b/test_app/djqscsv_tests/tests.py index 3ee7486..ba995b4 100644 --- a/test_app/djqscsv_tests/tests.py +++ b/test_app/djqscsv_tests/tests.py @@ -197,3 +197,31 @@ def test_empty_queryset(self): self.assertEqual(obj.getvalue(), '\xef\xbb\xbfid,name,address,info\r\n') + +class OrderingTests(CSVTestCase): + def setUp(self): + self.qs = create_people_and_get_queryset().extra( + select={'Most Powerful':"info LIKE '%arch mage%'"}) + + self.csv_with_extra = [ + ['\xef\xbb\xbfID', 'Person\'s name', 'address', + 'Info on Person', 'Most Powerful'], + ['1', 'vetch', 'iffish', 'wizard', '0'], + ['2', 'nemmerle', 'roke', 'deceased arch mage', '1'], + ['3', 'ged', 'gont', 'former arch mage', '1']] + + self.custom_order_csv = [[row[0], row[4]] + row[1:4] + for row in self.csv_with_extra] + + def test_extra_select(self): + obj = StringIO() + djqscsv.write_csv(self.qs, obj) + csv_file = filter(None, obj.getvalue().split('\n')) + self.assertMatchesCsv(csv_file, self.csv_with_extra) + + def test_extra_select_ordering(self): + obj = StringIO() + djqscsv.write_csv(self.qs, obj, field_order=['id', 'Most Powerful']) + csv_file = filter(None, obj.getvalue().split('\n')) + self.assertMatchesCsv(csv_file, self.custom_order_csv) +