diff --git a/core/common/constants.py b/core/common/constants.py
index 856ecbaf3..e38972344 100644
--- a/core/common/constants.py
+++ b/core/common/constants.py
@@ -75,6 +75,7 @@
CSV_DEFAULT_LIMIT = 1000
SEARCH_PARAM = 'q'
INCLUDE_FACETS = 'HTTP_INCLUDEFACETS'
+SEARCH_LATEST_REPO_VERSION = 'HTTP_INCLUDESEARCHLATEST'
INCLUDE_SEARCH_STATS = 'HTTP_INCLUDESEARCHSTATS'
FACETS_ONLY = 'facetsOnly'
SEARCH_STATS_ONLY = 'searchStatsOnly'
diff --git a/core/common/mixins.py b/core/common/mixins.py
index 550c468e2..15bff216e 100644
--- a/core/common/mixins.py
+++ b/core/common/mixins.py
@@ -16,7 +16,8 @@
from core.common.constants import HEAD, ACCESS_TYPE_NONE, INCLUDE_FACETS, \
LIST_DEFAULT_LIMIT, HTTP_COMPRESS_HEADER, CSV_DEFAULT_LIMIT, FACETS_ONLY, INCLUDE_RETIRED_PARAM, \
- SEARCH_STATS_ONLY, INCLUDE_SEARCH_STATS, UPDATED_BY_USERNAME_PARAM, CHECKSUM_STANDARD_HEADER, CHECKSUM_SMART_HEADER
+ SEARCH_STATS_ONLY, INCLUDE_SEARCH_STATS, UPDATED_BY_USERNAME_PARAM, CHECKSUM_STANDARD_HEADER, \
+ CHECKSUM_SMART_HEADER, SEARCH_LATEST_REPO_VERSION
from core.common.permissions import HasPrivateAccess, HasOwnership, CanViewConceptDictionary, \
CanViewConceptDictionaryVersion
from .checksums import ChecksumModel, Checksum
@@ -222,6 +223,9 @@ def serialize_list(self, results, paginator=None):
def should_include_facets(self):
return self.request.META.get(INCLUDE_FACETS, False) in TRUTHY
+ def is_latest_repo_search_header_present(self):
+ return self.request.META.get(SEARCH_LATEST_REPO_VERSION, False) in TRUTHY
+
def should_include_search_stats(self):
return self.request.META.get(INCLUDE_SEARCH_STATS, False) in TRUTHY
@@ -489,20 +493,6 @@ def latest_source_version(self):
def _cached_latest_source_version(self):
return self.parent.get_latest_released_version()
- def get_all_checksums(self):
- return {
- **super().get_all_checksums(),
- 'repo_versions': self.source_versions_checksum,
- }
-
- def set_source_versions_checksum(self):
- self.set_specific_checksums('repo_versions', self.source_versions_checksum)
-
- @property
- def source_versions_checksum(self):
- checksums = [version.checksum for version in self.sources.exclude(version=HEAD)]
- return self.generate_checksum(checksums) if checksums else None
-
@staticmethod
def is_strictly_equal(instance1, instance2):
return instance1.get_checksums() == instance2.get_checksums()
diff --git a/core/common/views.py b/core/common/views.py
index 40a26597b..8928e935f 100644
--- a/core/common/views.py
+++ b/core/common/views.py
@@ -389,14 +389,14 @@ def get_kwargs_filters(self): # pylint: disable=too-many-branches
filters['collection_url'] = f"{filters['collection_owner_url']}collections/{self.kwargs['collection']}/"
if is_version_specified and self.kwargs['version'] != HEAD:
filters['collection_url'] += f"{self.kwargs['version']}/"
- if is_source_specified and not is_version_specified and not self.should_search_latest_released_repo():
+ if is_source_specified and not is_version_specified and not self.should_search_latest_repo():
filters['source_version'] = HEAD
return filters
def get_latest_version_filter_field_for_source_child(self):
query_latest = self.__should_query_latest_version()
if query_latest:
- return 'is_in_latest_source_version'
+ return 'is_in_latest_source_version' if self.should_search_latest_repo() else 'is_latest_version'
if not self.is_global_scope() and (
self.kwargs.get('version') == HEAD or not self.kwargs.get('version')
) and 'collection' not in self.kwargs:
@@ -422,6 +422,8 @@ def get_facets(self):
raise Http400(detail=get(ex, 'info') or get(ex, 'error') or str(ex)) from ex
if not get(self.request.user, 'is_authenticated'):
facets.pop('updatedBy', None)
+ if self.should_search_latest_repo() and self.is_source_child_document_model() and 'source_version' in facets:
+ facets['source_version'] = [facet for facet in facets['source_version'] if facet[0] != 'HEAD']
return facets
def get_extras_searchable_fields_from_query_params(self):
@@ -470,7 +472,10 @@ def is_owner_document_model(self):
def is_source_child_document_model(self):
from core.concepts.documents import ConceptDocument
from core.mappings.documents import MappingDocument
- return self.document_model in [ConceptDocument, MappingDocument]
+ from core.concepts.search import ConceptFacetedSearch
+ from core.mappings.search import MappingFacetedSearch
+ return self.document_model in [
+ ConceptDocument, MappingDocument] or self.facet_class in [ConceptFacetedSearch, MappingFacetedSearch]
def is_concept_container_document_model(self):
from core.collections.documents import CollectionDocument
@@ -773,11 +778,12 @@ def should_perform_es_search(self):
bool(self.get_search_string()) or
self.has_searchable_extras_fields() or
bool(self.get_faceted_filters())
- ) or (SEARCH_PARAM in self.request.query_params.dict() and self.should_search_latest_released_repo())
+ ) or (SEARCH_PARAM in self.request.query_params.dict() and self.should_search_latest_repo())
- def should_search_latest_released_repo(self):
+ def should_search_latest_repo(self):
return self.is_source_child_document_model() and (
- 'version' not in self.kwargs and 'collection' not in self.kwargs)
+ 'version' not in self.kwargs and 'collection' not in self.kwargs
+ ) and self.is_latest_repo_search_header_present()
def has_searchable_extras_fields(self):
return bool(
diff --git a/core/integration_tests/tests_concepts.py b/core/integration_tests/tests_concepts.py
index c8618bc66..3d324b417 100644
--- a/core/integration_tests/tests_concepts.py
+++ b/core/integration_tests/tests_concepts.py
@@ -1627,7 +1627,7 @@ def setUp(self):
self.random_user = UserProfileFactory()
def test_search(self): # pylint: disable=too-many-statements
- response = self.client.get('/concepts/?q=MyConcept')
+ response = self.client.get('/concepts/?q=MyConcept2')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept2')
@@ -1636,34 +1636,157 @@ def test_search(self): # pylint: disable=too-many-statements
response = self.client.get('/concepts/?q=MyConcept1')
self.assertEqual(response.status_code, 200)
- self.assertEqual(len(response.data), 0)
+ self.assertEqual(len(response.data), 1)
+ self.assertEqual(response.data[0]['id'], 'MyConcept1')
+
+ response = self.client.get('/concepts/?q=MyConcept1&exact_match=on')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.data), 1)
+ self.assertEqual(response.data[0]['id'], 'MyConcept1')
response = self.client.get('/concepts/?q=MyConcept&conceptClass=classA')
self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.data), 1)
+ self.assertEqual(response.data[0]['id'], 'MyConcept1')
+
+ response = self.client.get('/concepts/?q=MyConcept1&conceptClass=classB')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.data), 0)
+
+ response = self.client.get('/concepts/?conceptClass=classA')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.data), 1)
+ self.assertEqual(response.data[0]['id'], 'MyConcept1')
+
+ response = self.client.get('/concepts/?extras.foo=bar')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.data), 1)
+ self.assertEqual(response.data[0]['id'], 'MyConcept1')
+
+ response = self.client.get('/concepts/?extras.exists=bar')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.data), 1)
+ self.assertEqual(response.data[0]['id'], 'MyConcept2')
+
+ response = self.client.get(
+ self.source.concepts_url + '?q=MyConcept&extras.exact.foo=bar&includeSearchMeta=true')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.data), 1)
+ self.assertEqual(response.data[0]['id'], 'MyConcept1')
+ self.assertEqual(
+ response.data[0]['search_meta']['search_highlight'],
+ {
+ 'extras.foo': ['bar'],
+ 'id': ['MyConcept1']
+ }
+ )
+
+ response = self.client.get(
+ self.source.uri + 'v1/concepts/?q=MyConcept&sortAsc=last_update',
+ HTTP_AUTHORIZATION='Token ' + self.random_user.get_token(),
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.data), 1)
+ self.assertEqual(response.data[0]['id'], 'MyConcept2')
+
+ response = self.client.get(
+ self.source.concepts_url + '?q=MyConcept&searchStatsOnly=true',
+ HTTP_AUTHORIZATION='Token ' + self.token,
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(
+ response.data,
+ [
+ {
+ 'name': 'high',
+ 'threshold': ANY,
+ 'confidence': ANY,
+ 'total': ANY
+ },
+ {
+ 'name': 'medium',
+ 'threshold': ANY,
+ 'confidence': ANY,
+ 'total': 0
+ },
+ {
+ 'name': 'low',
+ 'threshold': 0.01,
+ 'confidence': '<50.0%',
+ 'total': 0
+ }
+ ]
+ )
+ self.assertTrue(response.data[0]['total'] >= 2)
+
+ response = self.client.get(
+ self.source.concepts_url + '?q=MyConcept',
+ HTTP_AUTHORIZATION='Token ' + self.token,
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.data), 2)
+
+ def test_search_with_latest_released_repo_search(self): # pylint: disable=too-many-statements
+ response = self.client.get(
+ '/concepts/?q=MyConcept',
+ HTTP_INCLUDESEARCHLATEST=True,
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.data), 1)
+ self.assertEqual(response.data[0]['id'], 'MyConcept2')
+ self.assertEqual(response.data[0]['uuid'], str(self.concept2.get_latest_version().id))
+ self.assertEqual(response.data[0]['versioned_object_id'], self.concept2.id)
+
+ response = self.client.get(
+ '/concepts/?q=MyConcept1',
+ HTTP_INCLUDESEARCHLATEST=True,
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.data), 0)
+
+ response = self.client.get(
+ '/concepts/?q=MyConcept&conceptClass=classA',
+ HTTP_INCLUDESEARCHLATEST=True,
+ )
+ self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 0)
- response = self.client.get('/concepts/?q=MyConcept2&conceptClass=classB')
+ response = self.client.get(
+ '/concepts/?q=MyConcept2&conceptClass=classB',
+ HTTP_INCLUDESEARCHLATEST=True,
+ )
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept2')
self.assertEqual(response.data[0]['uuid'], str(self.concept2.get_latest_version().id))
self.assertEqual(response.data[0]['versioned_object_id'], self.concept2.id)
- response = self.client.get('/concepts/?conceptClass=classA')
+ response = self.client.get(
+ '/concepts/?conceptClass=classA',
+ HTTP_INCLUDESEARCHLATEST=True,
+ )
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 0)
- response = self.client.get('/concepts/?extras.foo=bar')
+ response = self.client.get(
+ '/concepts/?extras.foo=bar',
+ HTTP_INCLUDESEARCHLATEST=True,
+ )
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 0)
- response = self.client.get('/concepts/?extras.exists=bar')
+ response = self.client.get(
+ '/concepts/?extras.exists=bar',
+ HTTP_INCLUDESEARCHLATEST=True,
+ )
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept2')
response = self.client.get(
- self.source.concepts_url + '?q=MyConcept&extras.exact.bar=foo&includeSearchMeta=true')
+ self.source.concepts_url + '?q=MyConcept&extras.exact.bar=foo&includeSearchMeta=true',
+ HTTP_INCLUDESEARCHLATEST=True,
+ )
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]['id'], 'MyConcept2')
@@ -1675,6 +1798,7 @@ def test_search(self): # pylint: disable=too-many-statements
response = self.client.get(
self.source.uri + 'v1/concepts/?q=MyConcept&sortAsc=last_update',
HTTP_AUTHORIZATION='Token ' + self.random_user.get_token(),
+ HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
@@ -1683,6 +1807,7 @@ def test_search(self): # pylint: disable=too-many-statements
response = self.client.get(
self.source.concepts_url + '?q=MyConcept&searchStatsOnly=true',
HTTP_AUTHORIZATION='Token ' + self.token,
+ HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(
@@ -1698,6 +1823,7 @@ def test_search(self): # pylint: disable=too-many-statements
response = self.client.get(
self.source.concepts_url + '?q=MyConcept', # assumes HEAD
HTTP_AUTHORIZATION='Token ' + self.token,
+ HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 2)
@@ -1707,15 +1833,19 @@ def test_search(self): # pylint: disable=too-many-statements
response = self.client.get(
self.source.uri + 'HEAD/concepts/?q=MyConcept',
HTTP_AUTHORIZATION='Token ' + self.token,
+ HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 2)
- self.assertEqual(response.data[0]['id'], 'MyConcept1')
- self.assertEqual(response.data[1]['id'], 'MyConcept2')
+ self.assertEqual(
+ sorted([data['id'] for data in response.data]),
+ sorted(['MyConcept1', 'MyConcept2'])
+ )
response = self.client.get(
self.source.uri + 'v1/concepts/?q=MyConcept',
HTTP_AUTHORIZATION='Token ' + self.token,
+ HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
@@ -1731,6 +1861,28 @@ def test_facets(self):
self.assertEqual(response.status_code, 200)
self.assertEqual(list(response.data.keys()), ['facets'])
+ class_a_facet = [x for x in response.data['facets']['fields']['conceptClass'] if x[0] == 'classa'][0]
+ self.assertEqual(class_a_facet[0], 'classa')
+ self.assertTrue(class_a_facet[1] >= 1)
+ self.assertFalse(class_a_facet[2])
+
+ class_b_facet = [x for x in response.data['facets']['fields']['conceptClass'] if x[0] == 'classb'][0]
+ self.assertEqual(class_b_facet[0], 'classb')
+ self.assertTrue(class_b_facet[1] >= 1)
+ self.assertFalse(class_b_facet[2])
+
+ def test_facets_with_latest_released_repo_search(self):
+ if settings.ENV == 'ci':
+ rebuild_indexes(['concepts'])
+ ConceptDocument().update(self.source.concepts_set.all())
+
+ response = self.client.get(
+ '/concepts/?facetsOnly=true',
+ HTTP_INCLUDESEARCHLATEST=True,
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(list(response.data.keys()), ['facets'])
+
class_b_facet = [x for x in response.data['facets']['fields']['conceptClass'] if x[0] == 'classb'][0]
self.assertEqual(class_b_facet[0], 'classb')
self.assertTrue(class_b_facet[1] >= 1)
@@ -1738,7 +1890,8 @@ def test_facets(self):
self.assertEqual([x for x in response.data['facets']['fields']['conceptClass'] if x[0] == 'classa'], [])
response = self.client.get(
- self.source.uri + 'HEAD/concepts/?facetsOnly=true'
+ self.source.uri + 'HEAD/concepts/?facetsOnly=true',
+ HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(list(response.data.keys()), ['facets'])
@@ -1754,7 +1907,8 @@ def test_facets(self):
self.assertFalse(class_b_facet[2])
response = self.client.get(
- self.source.concepts_url + '?facetsOnly=true' # assumes HEAD
+ self.source.concepts_url + '?facetsOnly=true', # assumes HEAD
+ HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(list(response.data.keys()), ['facets'])
@@ -1769,7 +1923,8 @@ def test_facets(self):
self.assertFalse(class_a_facet[2])
response = self.client.get(
- self.source.uri + 'v1/concepts/?facetsOnly=true'
+ self.source.uri + 'v1/concepts/?facetsOnly=true',
+ HTTP_INCLUDESEARCHLATEST=True,
)
self.assertEqual(response.status_code, 200)
self.assertEqual(list(response.data.keys()), ['facets'])
diff --git a/core/settings.py b/core/settings.py
index 45f9f4b6e..c7dac3285 100644
--- a/core/settings.py
+++ b/core/settings.py
@@ -41,6 +41,7 @@
CORS_ALLOW_HEADERS = default_headers + (
'INCLUDEFACETS',
'INCLUDESEARCHSTATS',
+ 'INCLUDESEARCHLATEST'
)
CORS_EXPOSE_HEADERS = (