Skip to content

Commit

Permalink
OpenConceptLab/ocl_issues#1710 | include latest search by header
Browse files Browse the repository at this point in the history
  • Loading branch information
@snyaggarwal
snyaggarwal committed Dec 11, 2023
1 parent 86a8173 commit f3480ca
Show file tree
Hide file tree
Showing 5 changed files with 186 additions and 33 deletions.
1 change: 1 addition & 0 deletions core/common/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
CSV_DEFAULT_LIMIT = 1000
SEARCH_PARAM = 'q'
INCLUDE_FACETS = 'HTTP_INCLUDEFACETS'
SEARCH_LATEST_REPO_VERSION = 'HTTP_INCLUDESEARCHLATEST'
INCLUDE_SEARCH_STATS = 'HTTP_INCLUDESEARCHSTATS'
FACETS_ONLY = 'facetsOnly'
SEARCH_STATS_ONLY = 'searchStatsOnly'
Expand Down
20 changes: 5 additions & 15 deletions core/common/mixins.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@

from core.common.constants import HEAD, ACCESS_TYPE_NONE, INCLUDE_FACETS, \
LIST_DEFAULT_LIMIT, HTTP_COMPRESS_HEADER, CSV_DEFAULT_LIMIT, FACETS_ONLY, INCLUDE_RETIRED_PARAM, \
SEARCH_STATS_ONLY, INCLUDE_SEARCH_STATS, UPDATED_BY_USERNAME_PARAM, CHECKSUM_STANDARD_HEADER, CHECKSUM_SMART_HEADER
SEARCH_STATS_ONLY, INCLUDE_SEARCH_STATS, UPDATED_BY_USERNAME_PARAM, CHECKSUM_STANDARD_HEADER, \
CHECKSUM_SMART_HEADER, SEARCH_LATEST_REPO_VERSION
from core.common.permissions import HasPrivateAccess, HasOwnership, CanViewConceptDictionary, \
CanViewConceptDictionaryVersion
from .checksums import ChecksumModel, Checksum
Expand Down Expand Up @@ -222,6 +223,9 @@ def serialize_list(self, results, paginator=None):
def should_include_facets(self):
return self.request.META.get(INCLUDE_FACETS, False) in TRUTHY

def is_latest_repo_search_header_present(self):
return self.request.META.get(SEARCH_LATEST_REPO_VERSION, False) in TRUTHY

def should_include_search_stats(self):
return self.request.META.get(INCLUDE_SEARCH_STATS, False) in TRUTHY

Expand Down Expand Up @@ -489,20 +493,6 @@ def latest_source_version(self):
def _cached_latest_source_version(self):
return self.parent.get_latest_released_version()

def get_all_checksums(self):
return {
**super().get_all_checksums(),
'repo_versions': self.source_versions_checksum,
}

def set_source_versions_checksum(self):
self.set_specific_checksums('repo_versions', self.source_versions_checksum)

@property
def source_versions_checksum(self):
checksums = [version.checksum for version in self.sources.exclude(version=HEAD)]
return self.generate_checksum(checksums) if checksums else None

@staticmethod
def is_strictly_equal(instance1, instance2):
return instance1.get_checksums() == instance2.get_checksums()
Expand Down
18 changes: 12 additions & 6 deletions core/common/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,14 +389,14 @@ def get_kwargs_filters(self): # pylint: disable=too-many-branches
filters['collection_url'] = f"{filters['collection_owner_url']}collections/{self.kwargs['collection']}/"
if is_version_specified and self.kwargs['version'] != HEAD:
filters['collection_url'] += f"{self.kwargs['version']}/"
if is_source_specified and not is_version_specified and not self.should_search_latest_released_repo():
if is_source_specified and not is_version_specified and not self.should_search_latest_repo():
filters['source_version'] = HEAD
return filters

def get_latest_version_filter_field_for_source_child(self):
query_latest = self.__should_query_latest_version()
if query_latest:
return 'is_in_latest_source_version'
return 'is_in_latest_source_version' if self.should_search_latest_repo() else 'is_latest_version'
if not self.is_global_scope() and (
self.kwargs.get('version') == HEAD or not self.kwargs.get('version')
) and 'collection' not in self.kwargs:
Expand All @@ -422,6 +422,8 @@ def get_facets(self):
raise Http400(detail=get(ex, 'info') or get(ex, 'error') or str(ex)) from ex
if not get(self.request.user, 'is_authenticated'):
facets.pop('updatedBy', None)
if self.should_search_latest_repo() and self.is_source_child_document_model() and 'source_version' in facets:
facets['source_version'] = [facet for facet in facets['source_version'] if facet[0] != 'HEAD']
return facets

def get_extras_searchable_fields_from_query_params(self):
Expand Down Expand Up @@ -470,7 +472,10 @@ def is_owner_document_model(self):
def is_source_child_document_model(self):
from core.concepts.documents import ConceptDocument
from core.mappings.documents import MappingDocument
return self.document_model in [ConceptDocument, MappingDocument]
from core.concepts.search import ConceptFacetedSearch
from core.mappings.search import MappingFacetedSearch
return self.document_model in [
ConceptDocument, MappingDocument] or self.facet_class in [ConceptFacetedSearch, MappingFacetedSearch]

def is_concept_container_document_model(self):
from core.collections.documents import CollectionDocument
Expand Down Expand Up @@ -773,11 +778,12 @@ def should_perform_es_search(self):
bool(self.get_search_string()) or
self.has_searchable_extras_fields() or
bool(self.get_faceted_filters())
) or (SEARCH_PARAM in self.request.query_params.dict() and self.should_search_latest_released_repo())
) or (SEARCH_PARAM in self.request.query_params.dict() and self.should_search_latest_repo())

def should_search_latest_released_repo(self):
def should_search_latest_repo(self):
return self.is_source_child_document_model() and (
'version' not in self.kwargs and 'collection' not in self.kwargs)
'version' not in self.kwargs and 'collection' not in self.kwargs
) and self.is_latest_repo_search_header_present()

def has_searchable_extras_fields(self):
return bool(
Expand Down
179 changes: 167 additions & 12 deletions core/integration_tests/tests_concepts.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1627,7 +1627,7 @@ def setUp(self):
self.random_user = UserProfileFactory()

def test_search(self): # pylint: disable=too-many-statements
response = self.client.get('/concepts/?q=MyConcept')
response = self.client.get('/concepts/?q=MyConcept2')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept2')
Expand All @@ -1636,34 +1636,157 @@ def test_search(self): # pylint: disable=too-many-statements

response = self.client.get('/concepts/?q=MyConcept1')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 0)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept1')

response = self.client.get('/concepts/?q=MyConcept1&exact_match=on')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept1')

response = self.client.get('/concepts/?q=MyConcept&conceptClass=classA')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept1')

response = self.client.get('/concepts/?q=MyConcept1&conceptClass=classB')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 0)

response = self.client.get('/concepts/?conceptClass=classA')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept1')

response = self.client.get('/concepts/?extras.foo=bar')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept1')

response = self.client.get('/concepts/?extras.exists=bar')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept2')

response = self.client.get(
self.source.concepts_url + '?q=MyConcept&extras.exact.foo=bar&includeSearchMeta=true')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept1')
self.assertEqual(
response.data[0]['search_meta']['search_highlight'],
{
'extras.foo': ['<em>bar</em>'],
'id': ['<em>MyConcept1</em>']
}
)

response = self.client.get(
self.source.uri + 'v1/concepts/?q=MyConcept&sortAsc=last_update',
HTTP_AUTHORIZATION='Token ' + self.random_user.get_token(),
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept2')

response = self.client.get(
self.source.concepts_url + '?q=MyConcept&searchStatsOnly=true',
HTTP_AUTHORIZATION='Token ' + self.token,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.data,
[
{
'name': 'high',
'threshold': ANY,
'confidence': ANY,
'total': ANY
},
{
'name': 'medium',
'threshold': ANY,
'confidence': ANY,
'total': 0
},
{
'name': 'low',
'threshold': 0.01,
'confidence': '<50.0%',
'total': 0
}
]
)
self.assertTrue(response.data[0]['total'] >= 2)

response = self.client.get(
self.source.concepts_url + '?q=MyConcept',
HTTP_AUTHORIZATION='Token ' + self.token,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 2)

def test_search_with_latest_released_repo_search(self): # pylint: disable=too-many-statements
response = self.client.get(
'/concepts/?q=MyConcept',
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept2')
self.assertEqual(response.data[0]['uuid'], str(self.concept2.get_latest_version().id))
self.assertEqual(response.data[0]['versioned_object_id'], self.concept2.id)

response = self.client.get(
'/concepts/?q=MyConcept1',
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 0)

response = self.client.get(
'/concepts/?q=MyConcept&conceptClass=classA',
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 0)

response = self.client.get('/concepts/?q=MyConcept2&conceptClass=classB')
response = self.client.get(
'/concepts/?q=MyConcept2&conceptClass=classB',
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept2')
self.assertEqual(response.data[0]['uuid'], str(self.concept2.get_latest_version().id))
self.assertEqual(response.data[0]['versioned_object_id'], self.concept2.id)

response = self.client.get('/concepts/?conceptClass=classA')
response = self.client.get(
'/concepts/?conceptClass=classA',
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 0)

response = self.client.get('/concepts/?extras.foo=bar')
response = self.client.get(
'/concepts/?extras.foo=bar',
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 0)

response = self.client.get('/concepts/?extras.exists=bar')
response = self.client.get(
'/concepts/?extras.exists=bar',
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept2')

response = self.client.get(
self.source.concepts_url + '?q=MyConcept&extras.exact.bar=foo&includeSearchMeta=true')
self.source.concepts_url + '?q=MyConcept&extras.exact.bar=foo&includeSearchMeta=true',
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept2')
Expand All @@ -1675,6 +1798,7 @@ def test_search(self): # pylint: disable=too-many-statements
response = self.client.get(
self.source.uri + 'v1/concepts/?q=MyConcept&sortAsc=last_update',
HTTP_AUTHORIZATION='Token ' + self.random_user.get_token(),
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
Expand All @@ -1683,6 +1807,7 @@ def test_search(self): # pylint: disable=too-many-statements
response = self.client.get(
self.source.concepts_url + '?q=MyConcept&searchStatsOnly=true',
HTTP_AUTHORIZATION='Token ' + self.token,
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(
Expand All @@ -1698,6 +1823,7 @@ def test_search(self): # pylint: disable=too-many-statements
response = self.client.get(
self.source.concepts_url + '?q=MyConcept', # assumes HEAD
HTTP_AUTHORIZATION='Token ' + self.token,
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 2)
Expand All @@ -1707,15 +1833,19 @@ def test_search(self): # pylint: disable=too-many-statements
response = self.client.get(
self.source.uri + 'HEAD/concepts/?q=MyConcept',
HTTP_AUTHORIZATION='Token ' + self.token,
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 2)
self.assertEqual(response.data[0]['id'], 'MyConcept1')
self.assertEqual(response.data[1]['id'], 'MyConcept2')
self.assertEqual(
sorted([data['id'] for data in response.data]),
sorted(['MyConcept1', 'MyConcept2'])
)

response = self.client.get(
self.source.uri + 'v1/concepts/?q=MyConcept',
HTTP_AUTHORIZATION='Token ' + self.token,
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
Expand All @@ -1731,14 +1861,37 @@ def test_facets(self):
self.assertEqual(response.status_code, 200)
self.assertEqual(list(response.data.keys()), ['facets'])

class_a_facet = [x for x in response.data['facets']['fields']['conceptClass'] if x[0] == 'classa'][0]
self.assertEqual(class_a_facet[0], 'classa')
self.assertTrue(class_a_facet[1] >= 1)
self.assertFalse(class_a_facet[2])

class_b_facet = [x for x in response.data['facets']['fields']['conceptClass'] if x[0] == 'classb'][0]
self.assertEqual(class_b_facet[0], 'classb')
self.assertTrue(class_b_facet[1] >= 1)
self.assertFalse(class_b_facet[2])

def test_facets_with_latest_released_repo_search(self):
if settings.ENV == 'ci':
rebuild_indexes(['concepts'])
ConceptDocument().update(self.source.concepts_set.all())

response = self.client.get(
'/concepts/?facetsOnly=true',
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(list(response.data.keys()), ['facets'])

class_b_facet = [x for x in response.data['facets']['fields']['conceptClass'] if x[0] == 'classb'][0]
self.assertEqual(class_b_facet[0], 'classb')
self.assertTrue(class_b_facet[1] >= 1)
self.assertFalse(class_b_facet[2])
self.assertEqual([x for x in response.data['facets']['fields']['conceptClass'] if x[0] == 'classa'], [])

response = self.client.get(
self.source.uri + 'HEAD/concepts/?facetsOnly=true'
self.source.uri + 'HEAD/concepts/?facetsOnly=true',
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(list(response.data.keys()), ['facets'])
Expand All @@ -1754,7 +1907,8 @@ def test_facets(self):
self.assertFalse(class_b_facet[2])

response = self.client.get(
self.source.concepts_url + '?facetsOnly=true' # assumes HEAD
self.source.concepts_url + '?facetsOnly=true', # assumes HEAD
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(list(response.data.keys()), ['facets'])
Expand All @@ -1769,7 +1923,8 @@ def test_facets(self):
self.assertFalse(class_a_facet[2])

response = self.client.get(
self.source.uri + 'v1/concepts/?facetsOnly=true'
self.source.uri + 'v1/concepts/?facetsOnly=true',
HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(list(response.data.keys()), ['facets'])
Expand Down
1 change: 1 addition & 0 deletions core/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
CORS_ALLOW_HEADERS = default_headers + (
'INCLUDEFACETS',
'INCLUDESEARCHSTATS',
'INCLUDESEARCHLATEST'
)

CORS_EXPOSE_HEADERS = (
Expand Down

0 comments on commit f3480ca

Please sign in to comment.