Skip indexing for some files

refinery/data_set_manager/models.py

@@ -401,6 +401,12 @@ class Node(models.Model):
         METABOLITE_ASSIGNMENT_FILE
     }
 
+    INDEXED_FILES = {
+        RAW_DATA_FILE, DERIVED_DATA_FILE,
+        ARRAY_DATA_FILE, DERIVED_ARRAY_DATA_FILE,
+        ARRAY_DATA_MATRIX_FILE, DERIVED_ARRAY_DATA_MATRIX_FILE
+    }
+
     TYPES = ASSAYS | FILES | {
         SOURCE, SAMPLE, EXTRACT, LABELED_EXTRACT, SCAN, NORMALIZATION,
         DATA_TRANSFORMATION}

refinery/data_set_manager/search_indexes.py

@@ -10,6 +10,7 @@
 from django.conf import settings
 
 from haystack import indexes
+from haystack.exceptions import SkipDocument
 
 from file_store.models import FileStoreItem
 
@@ -71,6 +72,8 @@ def _assay_data(self, object):
     # https://groups.google.com/forum/?fromgroups#!topic/django-haystack/g39QjTkN-Yg
     # http://stackoverflow.com/questions/7399871/django-haystack-sort-results-by-title
     def prepare(self, object):
+        if object.type not in Node.INDEXED_FILES:
+            raise SkipDocument()
 
         data = super(NodeIndex, self).prepare(object)
         annotations = AnnotatedNode.objects.filter(node=object)

refinery/data_set_manager/tests.py

@@ -18,6 +18,8 @@
 from django.test import LiveServerTestCase, TestCase
 
 from guardian.shortcuts import assign_perm
+from haystack.exceptions import SkipDocument
+
 import mock
 from rest_framework.test import APIClient, APIRequestFactory, APITestCase
 
@@ -937,12 +939,6 @@ def test_generate_solr_params_no_params(self):
                          'Cell Line%2C'
                          'Type%2C'
                          'Group Name'
-                         '&fq=type%3A%28%22Raw Data File%22 '
-                         'OR %22Derived Data File%22 '
-                         'OR %22Array Data File%22 '
-                         'OR %22Derived Array Data File%22 '
-                         'OR %22Array Data Matrix File%22 '
-                         'OR%22Derived Array Data Matrix File%22%29'
                          '&fq=is_annotation%3Afalse'
                          '&start=0'
                          '&rows=10000000'
@@ -973,12 +969,6 @@ def test_generate_solr_params_for_assay_with_params(self):
                          '&facet.field=horse'
                          '&fl=cats%2Cmouse%2Cdog%2Chorse'
                          '&facet.pivot=cats%2Cmouse'
-                         '&fq=type%3A%28%22Raw Data File%22 '
-                         'OR %22Derived Data File%22 '
-                         'OR %22Array Data File%22 '
-                         'OR %22Derived Array Data File%22 '
-                         'OR %22Array Data Matrix File%22 '
-                         'OR%22Derived Array Data Matrix File%22%29'
                          '&fq=is_annotation%3Atrue'
                          '&start=2'
                          '&rows=7'
@@ -1919,7 +1909,8 @@ def setUp(self):
             assay=assay,
             study=study,
             file_uuid=self.file_store_item.uuid,
-            name='http://example.com/fake.txt'
+            name='http://example.com/fake.txt',
+            type='Raw Data File'
         )
 
         self.data_set_uuid = data_set.uuid
@@ -1933,6 +1924,11 @@ def setUp(self):
     def tearDown(self):
         FileStoreItem.objects.all().delete()
 
+    def test_skip_types(self):
+        self.node.type = 'Unknown File Type'
+        with self.assertRaises(SkipDocument):
+            NodeIndex().prepare(self.node)
+
     def test_prepare(self):
         data = NodeIndex().prepare(self.node)
         data = dict(
@@ -1961,7 +1957,7 @@ def test_prepare(self):
                 'REFINERY_FILETYPE_#_#_s': None,
                 'REFINERY_NAME_#_#_s': 'http://example.com/fake.txt',
                 'REFINERY_SUBANALYSIS_#_#_s': -1,
-                'REFINERY_TYPE_#_#_s': u'',
+                'REFINERY_TYPE_#_#_s': u'Raw Data File',
                 'REFINERY_WORKFLOW_OUTPUT_#_#_s': 'N/A',
                 'analysis_uuid': None,
                 'assay_uuid': self.assay_uuid,
@@ -1985,7 +1981,7 @@ def test_prepare(self):
                 'technology_Characteristics_generic_s': 'whizbang',
                 'technology_accession_Characteristics_generic_s': '',
                 'technology_source_Characteristics_generic_s': '',
-                'type': u'',
+                'type': u'Raw Data File',
                 'uuid': self.node_uuid,
                 'workflow_output': None
             }

refinery/data_set_manager/utils.py

@@ -637,13 +637,6 @@ def generate_solr_params(
     or None if assay_uuids is empty.
     """
 
-    file_types = 'fq=type:("Raw Data File" OR ' \
-                 '"Derived Data File" OR ' \
-                 '"Array Data File" OR ' \
-                 '"Derived Array Data File" OR ' \
-                 '"Array Data Matrix File" OR' \
-                 '"Derived Array Data Matrix File")'
-
     is_annotation = params.get('is_annotation', 'false')
     facet_count = params.get('include_facet_count', 'true')
     start = params.get('offset', '0')
@@ -657,8 +650,7 @@ def generate_solr_params(
     facet_filter = params.get('filter_attribute', None)
 
     fixed_solr_params = \
-        '&'.join([file_types,
-                  'fq=is_annotation:%s' % is_annotation,
+        '&'.join(['fq=is_annotation:%s' % is_annotation,
                   'start=%s' % start,
                   'rows=%s' % row,
                   'q=django_ct:data_set_manager.node&wt=json',

refinery/user_files_manager/tests.py

@@ -128,12 +128,6 @@ def test_generate_solr_params_for_user(self):
                          '%2Ctype'
                          '%2Cdjango_id'
                          '%2CREFINERY_DOWNLOAD_URL_s',
-                         'fq=type%3A%28%22Raw Data File%22 '
-                         'OR %22Derived Data File%22 '
-                         'OR %22Array Data File%22 '
-                         'OR %22Derived Array Data File%22 '
-                         'OR %22Array Data Matrix File%22 '
-                         'OR%22Derived Array Data Matrix File%22%29',
                          'fq=is_annotation%3Afalse',
                          'start=0',
                          'rows=10000000',