OpenConceptLab/ocl_issues#1635 | search with latest released repo ver…

…sion

core/common/mixins.py

@@ -15,7 +15,7 @@
 from rest_framework.response import Response
 
 from core.common.constants import HEAD, ACCESS_TYPE_NONE, INCLUDE_FACETS, \
-    LIST_DEFAULT_LIMIT, HTTP_COMPRESS_HEADER, CSV_DEFAULT_LIMIT, FACETS_ONLY, INCLUDE_RETIRED_PARAM,\
+    LIST_DEFAULT_LIMIT, HTTP_COMPRESS_HEADER, CSV_DEFAULT_LIMIT, FACETS_ONLY, INCLUDE_RETIRED_PARAM, \
     SEARCH_STATS_ONLY, INCLUDE_SEARCH_STATS
 from core.common.permissions import HasPrivateAccess, HasOwnership, CanViewConceptDictionary, \
     CanViewConceptDictionaryVersion
@@ -455,6 +455,21 @@ class SourceChildMixin(ChecksumModel):
     class Meta:
         abstract = True
 
+    @property
+    def is_in_latest_source_version(self):
+        version = self._cached_latest_source_version
+        return self.sources.filter(version=version.version).exists() if version else False
+
+    @property
+    def latest_source_version(self):
+        if self.is_in_latest_source_version:
+            return self._cached_latest_source_version
+        return None
+
+    @cached_property
+    def _cached_latest_source_version(self):
+        return self.parent.get_latest_released_version()
+
     def get_all_checksums(self):
         return {
             **super().get_all_checksums(),

core/common/search.py

@@ -24,7 +24,6 @@ def query(self, search, query):
                 return search.filter('query_string', fields=self.fields, query=search_str)
 
             return search.query('multi_match', query=search_str)
-
         return search
 
     def params(self, **kwargs):

core/common/views.py

@@ -211,7 +211,7 @@ def get_search_string(self, lower=True, decode=True):
 
     @property
     def is_fuzzy_search(self):
-        return self.request.query_params.dict().get('fuzzy', None) in get_truthy_values()
+        return self.request.query_params.dict().get('fuzzy', None) in TRUTHY
 
     def get_wildcard_search_string(self, _str):
         return CustomESSearch.get_wildcard_search_string(_str or self.get_search_string())
@@ -384,10 +384,13 @@ def get_kwargs_filters(self):  # pylint: disable=too-many-branches
                 filters['collection_url'] = f"{filters['collection_owner_url']}collections/{self.kwargs['collection']}/"
                 if is_version_specified and self.kwargs['version'] != HEAD:
                     filters['collection_url'] += f"{self.kwargs['version']}/"
-            if is_source_specified and not is_version_specified:
+            if is_source_specified and not is_version_specified and not self.should_search_latest_released_repo():
                 filters['source_version'] = HEAD
         return filters
 
+    def get_latest_version_filter_field_for_source_child(self):
+        return 'is_in_latest_source_version' if self.should_search_latest_released_repo() else 'is_latest_version'
+
     def get_facets(self):
         facets = {}
 
@@ -397,8 +400,8 @@ def get_facets(self):
             is_source_child_document_model = self.is_source_child_document_model()
             default_filters = self.default_filters.copy()
 
-            if is_source_child_document_model and 'collection' not in self.kwargs and 'version' not in self.kwargs:
-                default_filters['is_latest_version'] = True
+            if is_source_child_document_model and self.__should_query_latest_version():
+                default_filters[self.get_latest_version_filter_field_for_source_child()] = True
 
             faceted_filters = {to_camel_case(k): v for k, v in self.get_faceted_filters(True).items()}
             filters = {**default_filters, **self.get_facet_filters_from_kwargs(), **faceted_filters, 'retired': False}
@@ -413,7 +416,7 @@ def get_facets(self):
             try:
                 facets = faceted_search.execute().facets.to_dict()
             except TransportError as ex:  # pragma: no cover
-                raise Http400(detail=get(ex, 'error') or str(ex)) from ex
+                raise Http400(detail='Data too large.') from ex
 
         return facets
 
@@ -517,7 +520,7 @@ def __apply_common_search_filters(self):
         if self.is_user_document() and self.should_include_inactive():
             default_filters.pop('is_active', None)
         if self.is_source_child_document_model() and self.__should_query_latest_version():
-            default_filters['is_latest_version'] = True
+            default_filters[self.get_latest_version_filter_field_for_source_child()] = True
 
         for field, value in default_filters.items():
             results = results.query("match", **{field: value})
@@ -561,7 +564,7 @@ def __get_fuzzy_search_results(
         if sort:
             results = results.sort(*self._get_sort_attribute())
 
-        if self.request.query_params.get(INCLUDE_SEARCH_META_PARAM) in get_truthy_values():
+        if self.request.query_params.get(INCLUDE_SEARCH_META_PARAM) in TRUTHY:
             results = results.highlight(
                 *self.clean_fields_for_highlight(set(compact(self.get_wildcard_search_fields().keys()))))
 
@@ -663,8 +666,9 @@ def __search_results(self):  # pylint: disable=too-many-branches,too-many-locals
             else:
                 results = results.query('match', **{attr: value})
 
-        if self.request.query_params.get(INCLUDE_SEARCH_META_PARAM) in get_truthy_values():
+        if self.request.query_params.get(INCLUDE_SEARCH_META_PARAM) in TRUTHY:
             results = results.highlight(*self.clean_fields_for_highlight(fields))
+
         return results.sort(*self._get_sort_attribute())
 
     @staticmethod
@@ -716,7 +720,18 @@ def get_search_stats(
         ).get_aggregations(self.is_verbose(), self.is_raw())
 
     def should_perform_es_search(self):
-        return bool(self.get_search_string()) or self.has_searchable_extras_fields() or bool(self.get_faceted_filters())
+        return (
+                bool(self.get_search_string()) or
+                self.has_searchable_extras_fields() or
+                bool(self.get_faceted_filters())
+        ) or self.should_search_latest_released_repo()
+
+    def should_search_latest_released_repo(self):
+        return bool(
+            self.is_source_child_document_model() and
+            not self.request.user.is_anonymous and
+            self.request.user.should_search_released
+        )
 
     def has_searchable_extras_fields(self):
         return bool(
@@ -1065,4 +1080,4 @@ def post(_):
                     'state': mapping_task.state,
                 }
             }
-        )
+        )

core/concepts/documents.py

@@ -33,6 +33,7 @@ class Index:
     concept_class = fields.KeywordField(attr='concept_class', normalizer="lowercase")
     retired = fields.KeywordField(attr='retired')
     is_latest_version = fields.KeywordField(attr='is_latest_version')
+    is_in_latest_source_version = fields.KeywordField(attr='is_in_latest_source_version')
     extras = fields.ObjectField(dynamic=True)
     created_by = fields.KeywordField(attr='created_by.username')
     name_types = fields.ListField(fields.KeywordField())

core/concepts/models.py

@@ -237,6 +237,7 @@ class Meta:
         '_name': {'sortable': True, 'filterable': False, 'exact': False},
         'last_update': {'sortable': True, 'filterable': False, 'default': 'desc'},
         'is_latest_version': {'sortable': False, 'filterable': True},
+        'is_in_latest_source_version': {'sortable': False, 'filterable': True},
         'concept_class': {'sortable': True, 'filterable': True, 'facet': True, 'exact': False},
         'datatype': {'sortable': True, 'filterable': True, 'facet': True, 'exact': False},
         'locale': {'sortable': False, 'filterable': True, 'facet': True, 'exact': False},

core/concepts/search.py

@@ -10,7 +10,7 @@ class ConceptFacetedSearch(CustomESFacetedSearch):
     doc_types = [Concept]
     fields = [
         'datatype', 'concept_class', 'locale', 'retired',
-        'source', 'owner', 'owner_type', 'is_latest_version', 'is_active', 'name', 'collection', 'name_types',
+        'source', 'owner', 'owner_type', 'name', 'collection', 'name_types',
         'description_types', 'id', 'synonyms', 'extras'
     ]
 
@@ -23,8 +23,8 @@ class ConceptFacetedSearch(CustomESFacetedSearch):
         'collection': TermsFacet(field='collection', size=FACET_SIZE),
         'owner': TermsFacet(field='owner', size=FACET_SIZE),
         'ownerType': TermsFacet(field='owner_type'),
-        'is_active': TermsFacet(field='is_active'),
         'is_latest_version': TermsFacet(field='is_latest_version'),
+        'is_in_latest_source_version': TermsFacet(field='is_in_latest_source_version'),
         'collection_owner_url': TermsFacet(field='collection_owner_url', size=FACET_SIZE),
         'expansion': TermsFacet(field='expansion', size=FACET_SIZE),
         'nameTypes': TermsFacet(field='name_types', size=FACET_SIZE),

core/concepts/serializers.py

@@ -118,13 +118,14 @@ class ConceptAbstractSerializer(AbstractResourceSerializer):
     child_concept_urls = ListField(read_only=True)
     summary = SerializerMethodField()
     references = SerializerMethodField()
+    latest_source_version = CharField(source='latest_source_version.version', allow_null=True, allow_blank=True)
 
     class Meta:
         model = Concept
         abstract = True
         fields = AbstractResourceSerializer.Meta.fields + (
             'uuid', 'parent_concept_urls', 'child_concept_urls', 'parent_concepts', 'child_concepts', 'hierarchy_path',
-            'mappings', 'extras', 'summary', 'references', 'has_children'
+            'mappings', 'extras', 'summary', 'references', 'has_children', 'latest_source_version'
         )
 
     def __init__(self, *args, **kwargs):  # pylint: disable=too-many-branches
@@ -151,6 +152,8 @@ def __init__(self, *args, **kwargs):  # pylint: disable=too-many-branches
         is_verbose = self.__class__ == ConceptDetailSerializer
 
         try:
+            if request.user.is_anonymous or not request.user.should_search_released:
+                self.fields.pop('latest_source_version')
             if not self.include_parent_concepts:
                 self.fields.pop('parent_concepts', None)
             if not self.include_child_concepts:

core/concepts/views.py

@@ -13,6 +13,7 @@
 
 from core.bundles.models import Bundle
 from core.bundles.serializers import BundleSerializer
+from core.collections.documents import CollectionDocument
 from core.common.constants import (
     HEAD, INCLUDE_INVERSE_MAPPINGS_PARAM, INCLUDE_RETIRED_PARAM, ACCESS_TYPE_NONE)
 from core.common.exceptions import Http400, Http403
@@ -243,6 +244,8 @@ def get_object(self, queryset=None):
 
 
 class ConceptCollectionMembershipView(ConceptBaseView, ListWithHeadersMixin):
+    document_model = CollectionDocument
+
     def get_serializer_class(self):
         from core.collections.serializers import CollectionVersionListSerializer
         return CollectionVersionListSerializer

core/fixtures/auth_groups.yaml

@@ -14,3 +14,7 @@
   pk: 4
   fields:
     name: operations_panel
+- model: "auth.group"
+  pk: 5
+  fields:
+    name: search_released

core/users/constants.py

@@ -9,5 +9,8 @@
 OCL_FHIR_SERVERS_GROUP = 'ocl_fhir_servers'
 HAPI_FHIR_SERVERS_GROUP = 'hapi_fhir_servers'
 OPERATIONS_PANEL_GROUP = 'operations_panel'
-AUTH_GROUPS = [OCL_SERVERS_GROUP, OCL_FHIR_SERVERS_GROUP, HAPI_FHIR_SERVERS_GROUP, OPERATIONS_PANEL_GROUP]
+SEARCH_RELEASED_GROUP = 'search_released'
+AUTH_GROUPS = [
+    OCL_SERVERS_GROUP, OCL_FHIR_SERVERS_GROUP, HAPI_FHIR_SERVERS_GROUP, OPERATIONS_PANEL_GROUP, SEARCH_RELEASED_GROUP
+]
 INVALID_AUTH_GROUP_NAME = 'Invalid auth group.'

core/users/models.py

@@ -11,7 +11,7 @@
 from core.common.models import BaseModel, CommonLogoModel
 from core.common.tasks import send_user_verification_email, send_user_reset_password_email
 from core.common.utils import web_url
-from core.users.constants import AUTH_GROUPS
+from core.users.constants import AUTH_GROUPS, SEARCH_RELEASED_GROUP
 from .constants import USER_OBJECT_TYPE
 
 
@@ -175,6 +175,13 @@ def is_valid_auth_group(*names):
     def auth_groups(self):
         return self.groups.values_list('name', flat=True)
 
+    @property
+    def should_search_released(self):
+        return self.is_staff or self.has_auth_group(SEARCH_RELEASED_GROUP)
+
+    def has_auth_group(self, group_name):
+        return self.groups.filter(name=group_name).exists()
+
     @property
     def auth_headers(self):
         return {'Authorization': f'Token {self.get_token()}'}