Merge pull request #175 from cbmi/issue-174

Update behavior of exporter force_distinct to truly be distinct

avocado/export/__init__.py

@@ -1,5 +1,6 @@
 from avocado.core import loader
 from avocado.conf import OPTIONAL_DEPS
+from _base import BaseExporter  # noqa
 from _csv import CSVExporter
 from _sas import SASExporter
 from _r import RExporter

avocado/export/_base.py

@@ -62,16 +62,26 @@ def read(self, iterable, force_distinct=True, *args, **kwargs):
         """Takes an iterable that produces rows to be formatted.
 
         If `force_distinct` is true, rows will be filtered based on the slice
-        of the row that is *up* for to be formatted. Note, this assumes the
-        rows are ordered.
+        of the row that is going to be formatted.
         """
-        last_row = None
+        unique_rows = set()
+
         for row in iterable:
             _row = row[:self.row_length]
-            if force_distinct and _row == last_row:
-                continue
-            last_row = _row
+
+            if force_distinct:
+                _row_hash = hash(tuple(_row))
+
+                if _row_hash in unique_rows:
+                    continue
+
+                unique_rows.add(_row_hash)
+
             yield self._format_row(_row, **kwargs)
 
     def write(self, iterable, *args, **kwargs):
-        raise NotImplemented
+        for row_gen in self.read(iterable, *args, **kwargs):
+            row = []
+            for data in row_gen:
+                row.extend(data.values())
+            yield tuple(row)

avocado/models.py

@@ -46,7 +46,7 @@ class DataCategory(Base, PublishArchiveMixin):
     objects = managers.DataCategoryManager()
 
     class Meta(object):
-        ordering = ('-parent__id', 'order', 'name')
+        ordering = ('parent__order', 'parent__name', 'order', 'name')
         verbose_name_plural = 'data categories'
 
 
@@ -112,7 +112,7 @@ class DataField(BasePlural, PublishArchiveMixin):
 
     class Meta(object):
         unique_together = ('app_name', 'model_name', 'field_name')
-        ordering = ('name',)
+        ordering = ('category__order', 'category__name', 'order', 'name')
         permissions = (
             ('view_datafield', 'Can view datafield'),
         )
@@ -473,7 +473,7 @@ def format(self, *args, **kwargs):
 
     class Meta(object):
         app_label = 'avocado'
-        ordering = ('order',)
+        ordering = ('category__order', 'category__name', 'order', 'name')
         permissions = (
             ('view_dataconcept', 'Can view dataconcept'),
         )
@@ -492,7 +492,7 @@ class DataConceptField(models.Model):
     modified = models.DateTimeField(auto_now=True)
 
     class Meta(object):
-        ordering = ('order',)
+        ordering = ('order', 'name')
 
     def __unicode__(self):
         return unicode(self.name or self.field.name)

tests/cases/exporting/tests.py

@@ -8,7 +8,8 @@
 from avocado.models import DataField, DataConcept, DataConceptField, DataView
 from ... import models
 
-__all__ = ['FileExportTestCase', 'ResponseExportTestCase']
+__all__ = ['FileExportTestCase', 'ResponseExportTestCase',
+           'ForceDistinctRegressionTestCase']
 
 
 class ExportTestCase(TestCase):
@@ -175,3 +176,79 @@ def test_html(self):
 </table>""")
         exporter.write(self.query, template=template, buff=response)
         self.assertEqual(len(response.content), 494)
+
+
+class ForceDistinctRegressionTestCase(TestCase):
+    fixtures = ['employee_data.json']
+
+    def setUp(self):
+        management.call_command('avocado', 'init', 'tests', quiet=True)
+
+        self.first_name = DataField.objects.get(field_name='first_name')\
+            .concepts.all()[0]
+        self.last_name = DataField.objects.get(field_name='last_name')\
+            .concepts.all()[0]
+        self.project_name = DataField.objects\
+            .get(model_name='project', field_name='name')\
+            .concepts.all()[0]
+        self.title_name = DataField.objects\
+            .get(model_name='title', field_name='name')\
+            .concepts.all()[0]
+
+        e1 = models.Employee.objects.get(pk=1)
+        e2 = models.Employee.objects.get(pk=2)
+
+        p1 = models.Project(name='P1', manager=e1)
+        p1.save()
+
+        p2 = models.Project(name='P2', manager=e2)
+        p2.save()
+
+        p1.employees = [e1, e2]
+        p2.employees = [e1, e2]
+
+    def test_sort_flat(self):
+        "Sorts on a non-reverse foreign key property."
+        view = DataView(json=[
+            {'concept': self.first_name.pk},
+            {'concept': self.last_name.pk},
+            {'concept': self.title_name.pk, 'sort': 'desc', 'visible': False},
+        ])
+
+        queryset = view.apply()
+
+        exporter = export.BaseExporter(view)
+        exporter.params.insert(0, (RawFormatter(keys=['pk']), 1))
+        exporter.row_length += 1
+
+        self.assertEqual(list(exporter.write(queryset.raw())), [
+            (1, u'Eric', u'Smith'),
+            (3, u'Erick', u'Smith'),
+            (5, u'Zac', u'Cook'),
+            (2, u'Erin', u'Jones'),
+            (4, u'Aaron', u'Harris'),
+            (6, u'Mel', u'Brooks'),
+        ])
+
+    def test_sort_related(self):
+        "Sorts on a reverse foreign key property."
+        view = DataView(json=[
+            {'concept': self.first_name.pk},
+            {'concept': self.last_name.pk},
+            {'concept': self.project_name.pk, 'sort': 'asc', 'visible': False},
+        ])
+
+        queryset = view.apply()
+
+        exporter = export.BaseExporter(view)
+        exporter.params.insert(0, (RawFormatter(keys=['pk']), 1))
+        exporter.row_length += 1
+
+        self.assertEqual(list(exporter.write(queryset.raw())), [
+            (3, u'Erick', u'Smith'),
+            (4, u'Aaron', u'Harris'),
+            (5, u'Zac', u'Cook'),
+            (6, u'Mel', u'Brooks'),
+            (1, u'Eric', u'Smith'),
+            (2, u'Erin', u'Jones')
+        ])