From e98ac8ff75a64c2f759912985f81ddd094ede5b2 Mon Sep 17 00:00:00 2001 From: Michael Maurizi Date: Wed, 2 Nov 2016 11:02:24 -0400 Subject: [PATCH] Change `render_to_csv_response` to return a `StreamingHttpResponse`. The old behaviour is available by setting the keyword argument `streaming` to `False`. Also drop support for Python 2.6 and Django 1.5-1.7, adds Django 1.10 to the test matrix, and cleans up the README a bit. Fixes #86, Fixes #87 --- .travis.yml | 15 +---- README.rst | 18 +++++- djqscsv/djqscsv.py | 63 ++++++++++++------- setup.py | 2 +- .../djqscsv_tests/tests/test_csv_creation.py | 27 +++++--- 5 files changed, 76 insertions(+), 49 deletions(-) diff --git a/.travis.yml b/.travis.yml index 773dde1..16bf1ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,13 @@ language: python python: - - 2.6 - 2.7 - 3.5 env: - - DJANGO=1.5 - - DJANGO=1.6 - - DJANGO=1.7 - DJANGO=1.8 - DJANGO=1.9 + - DJANGO=1.10 matrix: exclude: - - python: 2.6 - env: DJANGO=1.6 - - python: 2.6 - env: DJANGO=1.7 - - python: 2.6 - env: DJANGO=1.8 - - python: 2.6 - env: DJANGO=1.9 - - python: 3.5 - env: DJANGO=1.5 - python: 3.5 env: DJANGO=1.6 - python: 3.5 diff --git a/README.rst b/README.rst index 59b9038..cb9b7b4 100644 --- a/README.rst +++ b/README.rst @@ -27,13 +27,13 @@ installation Run:: pip install django-queryset-csv - -Supports Python 2.6 and 2.7, Django >= 1.5. + +Supports Python 2.7 and 3.5, Django >= 1.8. usage ----- Perform all filtering and field authorization in your view using ``.filter()`` and ``.values()``. -Then, use ``render_to_csv_response`` to turn a queryset into a respone with a CSV attachment. +Then, use ``render_to_csv_response`` to turn a queryset into a response with a CSV attachment. Pass it a ``QuerySet`` or ``ValuesQuerySet`` instance:: from djqscsv import render_to_csv_response @@ -42,6 +42,14 @@ Pass it a ``QuerySet`` or ``ValuesQuerySet`` instance:: qs = Foo.objects.filter(bar=True).values('id', 'bar') return render_to_csv_response(qs) +If you need to write the CSV to a file you can use ``write_csv`` instead:: + + from djqscsv import write_csv + + qs = Foo.objects.filter(bar=True).values('id', 'bar') + with open('foo.csv', 'w') as csv_file: + write_csv(qs, csv_file) + foreign keys ------------ @@ -77,6 +85,10 @@ This module exports two functions that write CSVs, ``render_to_csv_response`` an - ``use_verbose_names`` - (default: ``True``) A boolean determining whether to use the django field's ``verbose_name``, or to use it's regular field name as a column header. Note that if a given field is found in the ``field_header_map``, this value will take precendence. - ``field_order`` - (default: ``None``) A list of fields to determine the sort order. This list need not be complete: any fields not specified will follow those in the list with the order they would have otherwise used. +In addition to the above arguments, ``render_to_csv_response`` takes the following optional keyword argument: + +- ``streaming`` - (default: ``True``) A boolean determining whether to use ``StreamingHttpResponse`` instead of the normal ``HttpResponse``. + The remaining keyword arguments are *passed through* to the csv writer. For example, you can export a CSV with a different delimiter. views.py:: diff --git a/djqscsv/djqscsv.py b/djqscsv/djqscsv.py index 3fb62af..a0afb05 100644 --- a/djqscsv/djqscsv.py +++ b/djqscsv/djqscsv.py @@ -4,7 +4,7 @@ from django.core.exceptions import ValidationError from django.utils.text import slugify -from django.http import HttpResponse +from django.http import HttpResponse, StreamingHttpResponse from django.utils import six @@ -12,16 +12,24 @@ # Keyword arguments that will be used by this module # the rest will be passed along to the csv writer -DJQSCSV_KWARGS = {'field_header_map': None, - 'field_serializer_map': None, - 'use_verbose_names': True, - 'field_order': None} +DJQSCSV_KWARGS = { + 'field_header_map', 'field_serializer_map', 'use_verbose_names', + 'field_order', 'streaming'} class CSVException(Exception): pass +class _Echo(object): + """An object that implements just the write method of the file-like + interface. + """ + def write(self, value): + """Write the value by returning it, instead of storing in a buffer.""" + return value + + def render_to_csv_response(queryset, filename=None, append_datestamp=False, **kwargs): """ @@ -36,19 +44,34 @@ def render_to_csv_response(queryset, filename=None, append_datestamp=False, filename = generate_filename(queryset, append_datestamp=append_datestamp) - response = HttpResponse(content_type='text/csv') + response_args = {'content_type': 'text/csv'} + + if not kwargs.get('streaming', True): + response = HttpResponse(**response_args) + write_csv(queryset, response, **kwargs) + else: + response = StreamingHttpResponse( + _iter_csv(queryset, _Echo(), **kwargs), **response_args) + response['Content-Disposition'] = 'attachment; filename=%s;' % filename response['Cache-Control'] = 'no-cache' - write_csv(queryset, response, **kwargs) - return response def write_csv(queryset, file_obj, **kwargs): + """ + Writes CSV data to a file object based on the contents of the queryset. + """ + # Force iteration over all rows so they all get written to the file + for _ in _iter_csv(queryset, file_obj, **kwargs): + pass + + +def _iter_csv(queryset, file_obj, **kwargs): """ The main worker function. Writes CSV data to a file object based on the - contents of the queryset. + contents of the queryset and yields each row. """ # process keyword arguments to pull out the ones used by this function @@ -64,7 +87,7 @@ def write_csv(queryset, file_obj, **kwargs): csv_kwargs[key] = val # add BOM to support CSVs in MS Excel (for Windows only) - file_obj.write(b'\xef\xbb\xbf') + yield file_obj.write(b'\xef\xbb\xbf') # the CSV must always be built from a values queryset # in order to introspect the necessary fields. @@ -86,25 +109,17 @@ def write_csv(queryset, file_obj, **kwargs): values_qs = queryset.values() try: + # Django 1.9+ field_names = values_qs.query.values_select except AttributeError: - try: - field_names = values_qs.field_names - except AttributeError: - # in django1.5, empty querysets trigger - # this exception, but not django 1.6 - raise CSVException("Empty queryset provided to exporter.") + # Django 1.8 + field_names = values_qs.field_names extra_columns = list(values_qs.query.extra_select) if extra_columns: field_names += extra_columns - try: - aggregate_columns = list(values_qs.query.annotation_select) - except AttributeError: - # this gets a deprecation warning in django 1.9 but is - # required in django<=1.7 - aggregate_columns = list(values_qs.query.aggregate_select) + aggregate_columns = list(values_qs.query.annotation_select) if aggregate_columns: field_names += aggregate_columns @@ -134,11 +149,11 @@ def write_csv(queryset, file_obj, **kwargs): merged_header_map.update(dict((k, k) for k in extra_columns)) merged_header_map.update(field_header_map) - writer.writerow(merged_header_map) + yield writer.writerow(merged_header_map) for record in values_qs: record = _sanitize_record(field_serializer_map, record) - writer.writerow(record) + yield writer.writerow(record) def generate_filename(queryset, append_datestamp=False): diff --git a/setup.py b/setup.py index 82c96ba..ee58390 100644 --- a/setup.py +++ b/setup.py @@ -23,5 +23,5 @@ "Framework :: Django", "License :: OSI Approved :: GNU General Public License (GPL)" ], - install_requires=['django>=1.5', 'unicodecsv>=0.14.1'], + install_requires=['django>=1.8', 'unicodecsv>=0.14.1'], ) diff --git a/test_app/djqscsv_tests/tests/test_csv_creation.py b/test_app/djqscsv_tests/tests/test_csv_creation.py index c2ae0b1..6b97732 100644 --- a/test_app/djqscsv_tests/tests/test_csv_creation.py +++ b/test_app/djqscsv_tests/tests/test_csv_creation.py @@ -298,8 +298,9 @@ def test_render_to_csv_response_no_filename(self): response = djqscsv.render_to_csv_response(self.qs, use_verbose_names=False) self.assertEqual(response['Content-Type'], 'text/csv') - self.assertMatchesCsv(response.content.splitlines(), - self.FULL_PERSON_CSV_NO_VERBOSE) + self.assertMatchesCsv( + b''.join(response.streaming_content).splitlines(), + self.FULL_PERSON_CSV_NO_VERBOSE) self.assertRegexpMatches(response['Content-Disposition'], r'attachment; filename=person_export.csv;') @@ -309,6 +310,16 @@ def test_render_to_csv_response(self): filename="test_csv", use_verbose_names=False) self.assertEqual(response['Content-Type'], 'text/csv') + self.assertMatchesCsv( + b''.join(response.streaming_content).splitlines(), + self.FULL_PERSON_CSV_NO_VERBOSE) + + def test_render_to_csv_response_non_streaming(self): + response = djqscsv.render_to_csv_response(self.qs, + filename="test_csv", + use_verbose_names=False, + streaming=False) + self.assertEqual(response['Content-Type'], 'text/csv') self.assertMatchesCsv(response.content.splitlines(), self.FULL_PERSON_CSV_NO_VERBOSE) @@ -319,9 +330,10 @@ def test_render_to_csv_response_other_delimiter(self): delimiter='|') self.assertEqual(response['Content-Type'], 'text/csv') - self.assertMatchesCsv(response.content.splitlines(), - self.FULL_PERSON_CSV_NO_VERBOSE, - delimiter="|") + self.assertMatchesCsv( + b''.join(response.streaming_content).splitlines(), + self.FULL_PERSON_CSV_NO_VERBOSE, + delimiter="|") def test_render_to_csv_fails_on_delimiter_mismatch(self): response = djqscsv.render_to_csv_response(self.qs, @@ -330,5 +342,6 @@ def test_render_to_csv_fails_on_delimiter_mismatch(self): delimiter='|') self.assertEqual(response['Content-Type'], 'text/csv') - self.assertNotMatchesCsv(response.content.splitlines(), - self.FULL_PERSON_CSV_NO_VERBOSE) + self.assertNotMatchesCsv( + b''.join(response.streaming_content).splitlines(), + self.FULL_PERSON_CSV_NO_VERBOSE)