Remove stop_words when building the primary dataset to reduce datas e…

…fficiently
Fantomas42 · May 3, 2015 · 10ab56f · 10ab56f
1 parent d8a367e
commit 10ab56f
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 7 deletions.
diff --git a/zinnia/comparison.py b/zinnia/comparison.py
@@ -8,6 +8,7 @@
 
 from zinnia.settings import F_MIN
 from zinnia.settings import F_MAX
+from zinnia.search import STOP_WORDS
 
 
 def pearson_score(list1, list2):
@@ -51,7 +52,7 @@ def dataset(self):
             item = list(item)
             item_pk = item.pop(0)
             datas = ' '.join(map(six.text_type, item))
-            dataset[item_pk] = strip_tags(datas).lower()
+            dataset[item_pk] = STOP_WORDS.rebase(strip_tags(datas).lower(), '')
         return dataset
 
 

diff --git a/zinnia/tests/test_comparison.py b/zinnia/tests/test_comparison.py
@@ -35,8 +35,8 @@ def test_clustered_model(self):
         cm = ClusteredModel(Entry.objects.all(),
                             ['title', 'excerpt', 'content'])
         self.assertEqual(sorted(cm.dataset().values()),
-                         sorted(['my entry 1  my content 1',
-                                 'my entry 2  my content 2']))
+                         sorted([' entry 1   content 1',
+                                 ' entry 2   content 2']))
 
     def test_vector_builder(self):
         vectors = VectorBuilder(Entry.objects.all(),
@@ -54,9 +54,8 @@ def test_vector_builder(self):
         self.assertEqual(vectors._dataset, {})
         self.assertEqual(vectors._columns, [])
         self.assertEqual(sorted(vectors.columns), sorted(
-            ['content', 'this', 'is', '1',
-             'second', '2', 'first']))
+            ['1', '2', 'content']))
         self.assertEqual(sorted([sorted(row) for row in
                                  vectors._dataset.values()]),
-                         sorted([sorted([1, 1, 1, 1, 0, 0, 1]),
-                                 sorted([0, 0, 0, 0, 1, 1, 0])]))
+                         sorted([sorted([0, 0, 1]),
+                                 sorted([0, 1, 1])]))