Skip to content

Commit

Permalink
[MSEARCH-784] add bibframe authority index
Browse files Browse the repository at this point in the history
  • Loading branch information
aleksei-pronichev committed Jun 21, 2024
1 parent e90e766 commit cf5d149
Show file tree
Hide file tree
Showing 29 changed files with 588 additions and 73 deletions.
1 change: 1 addition & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
* Remove ability to match on LCCN searches without a prefix ([MSEARCH-752](https://folio-org.atlassian.net/browse/MSEARCH-752))
* Search consolidated items/holdings data in consortium ([MSEARCH-759](https://folio-org.atlassian.net/browse/MSEARCH-759))
* Create bibframe index and process bibframe events ([MSEARCH-781](https://folio-org.atlassian.net/browse/MSEARCH-781))
* Create bibframe authority index and process bibframe authority events ([MSEARCH-784](https://folio-org.atlassian.net/browse/MSEARCH-784))
* Allow Unified List of Inventory Locations in a Consortium to be fetched by member tenants ([MSEARCH-660](https://folio-org.atlassian.net/browse/MSEARCH-660))
* Implement Indexing of Campuses from Kafka ([MSEARCH-770](https://issues.folio.org/browse/MSEARCH-770))
* Extend response with additional Location fields for Inventory Locations in a Consortium endpoint ([MSEARCH-775](https://folio-org.atlassian.net/browse/MSEARCH-775))
Expand Down
17 changes: 9 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -414,14 +414,15 @@ Consortium feature on module enable is defined by 'centralTenantId' tenant param

### Search API

| METHOD | URL | DESCRIPTION |
|:-------|:------------------------------|:-------------------------------------------------------------------------------------|
| GET | `/search/instances` | Search by instances and to this instance items and holding-records |
| GET | `/search/authorities` | Search by authority records |
| GET | `/search/bibframe` | Search linked data graph resource descriptions |
| GET | `/search/{recordType}/facets` | Get facets where recordType could be: instances, authorities, contributors, subjects |
| GET | ~~`/search/instances/ids`~~ | (DEPRECATED) Stream instance ids as JSON or plain text |
| GET | ~~`/search/holdings/ids`~~ | (DEPRECATED) Stream holding record ids as JSON or plain text |
| METHOD | URL | DESCRIPTION |
|:-------|:------------------------------------------|:-------------------------------------------------------------------------------------|
| GET | `/search/instances` | Search by instances and to this instance items and holding-records |
| GET | `/search/authorities` | Search by authority records |
| GET | `/search/bibframe` | Search linked data graph resource descriptions |
| GET | `/search/bibframe/authorities` | Search linked data graph authority resource descriptions |
| GET | `/search/{recordType}/facets` | Get facets where recordType could be: instances, authorities, contributors, subjects |
| GET | ~~`/search/instances/ids`~~ | (DEPRECATED) Stream instance ids as JSON or plain text |
| GET | ~~`/search/holdings/ids`~~ | (DEPRECATED) Stream holding record ids as JSON or plain text |

#### Searching and filtering

Expand Down
9 changes: 9 additions & 0 deletions descriptors/ModuleDescriptor-template.json
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,15 @@
"search.bibframe.collection.get"
]
},
{
"methods": [
"GET"
],
"pathPattern": "/search/bibframe/authorities",
"permissionsRequired": [
"search.bibframe.collection.get"
]
},
{
"methods": [
"GET"
Expand Down
19 changes: 19 additions & 0 deletions src/main/java/org/folio/search/controller/SearchController.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import org.folio.search.domain.dto.Authority;
import org.folio.search.domain.dto.AuthoritySearchResult;
import org.folio.search.domain.dto.Bibframe;
import org.folio.search.domain.dto.BibframeAuthority;
import org.folio.search.domain.dto.BibframeSearchAuthorityResult;
import org.folio.search.domain.dto.BibframeSearchResult;
import org.folio.search.domain.dto.Instance;
import org.folio.search.domain.dto.InstanceSearchResult;
Expand Down Expand Up @@ -65,6 +67,23 @@ public ResponseEntity<BibframeSearchResult> searchBibframe(String tenant, String
);
}

@Override
public ResponseEntity<BibframeSearchAuthorityResult> searchBibframeAuthorities(String tenant,
String query,
Integer limit,
Integer offset) {
var searchRequest = CqlSearchRequest.of(
BibframeAuthority.class, tenant, query, limit, offset, true);
var result = searchService.search(searchRequest);
return ResponseEntity.ok(new BibframeSearchAuthorityResult()
.searchQuery(query)
.content(result.getRecords())
.pageNumber(divPlusOneIfRemainder(offset, limit))
.totalPages(divPlusOneIfRemainder(result.getTotalRecords(), limit))
.totalRecords(result.getTotalRecords())
);
}

private int divPlusOneIfRemainder(int one, int two) {
var modulo = one % two;
return one / two + (modulo > 0 ? 1 : 0);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import static org.folio.search.utils.SearchConverterUtils.getEventPayload;
import static org.folio.search.utils.SearchConverterUtils.getResourceEventId;
import static org.folio.search.utils.SearchConverterUtils.getResourceSource;
import static org.folio.search.utils.SearchUtils.BIBFRAME_RESOURCE;
import static org.folio.search.utils.SearchUtils.ID_FIELD;
import static org.folio.search.utils.SearchUtils.INSTANCE_ID_FIELD;
import static org.folio.search.utils.SearchUtils.INSTANCE_RESOURCE;
Expand Down Expand Up @@ -206,7 +205,7 @@ public void handleBibframeEvents(List<ConsumerRecord<String, ResourceEvent>> con
log.info("Processing bibframe events from Kafka [number of events: {}]", consumerRecords.size());
var batch = consumerRecords.stream()
.map(ConsumerRecord::value)
.map(bibframe -> bibframe.resourceName(BIBFRAME_RESOURCE).id(getResourceEventId(bibframe)))
.map(bibframe -> bibframe.id(getResourceEventId(bibframe)))
.toList();

indexResources(batch, resourceService::indexResources);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,19 @@
@Component
public class ResourceEventBatchInterceptor implements BatchInterceptor<String, ResourceEvent> {

private static final Map<String, String> TOPIC_TO_RESOURCE_MAP = Map.of(
"inventory.instance", SearchUtils.INSTANCE_RESOURCE,
"inventory.holdings-record", SearchUtils.INSTANCE_RESOURCE,
"inventory.item", SearchUtils.INSTANCE_RESOURCE,
"inventory.bound-with", SearchUtils.INSTANCE_RESOURCE,
"authorities.authority", SearchUtils.AUTHORITY_RESOURCE,
"search.instance-contributor", SearchUtils.CONTRIBUTOR_RESOURCE,
"search.instance-subject", SearchUtils.INSTANCE_SUBJECT_RESOURCE,
"inventory.classification-type", SearchUtils.CLASSIFICATION_TYPE_RESOURCE,
"inventory.location", SearchUtils.LOCATION_RESOURCE,
"inventory.campus", SearchUtils.CAMPUS_RESOURCE
private static final Map<String, String> TOPIC_TO_RESOURCE_MAP = Map.ofEntries(
Map.entry("inventory.instance", SearchUtils.INSTANCE_RESOURCE),
Map.entry("inventory.holdings-record", SearchUtils.INSTANCE_RESOURCE),
Map.entry("inventory.item", SearchUtils.INSTANCE_RESOURCE),
Map.entry("inventory.bound-with", SearchUtils.INSTANCE_RESOURCE),
Map.entry("authorities.authority", SearchUtils.AUTHORITY_RESOURCE),
Map.entry("search.instance-contributor", SearchUtils.CONTRIBUTOR_RESOURCE),
Map.entry("search.instance-subject", SearchUtils.INSTANCE_SUBJECT_RESOURCE),
Map.entry("inventory.classification-type", SearchUtils.CLASSIFICATION_TYPE_RESOURCE),
Map.entry("inventory.location", SearchUtils.LOCATION_RESOURCE),
Map.entry("inventory.campus", SearchUtils.CAMPUS_RESOURCE),
Map.entry("search.bibframe", SearchUtils.BIBFRAME_RESOURCE),
Map.entry("search.bibframe-authorities", SearchUtils.BIBFRAME_AUTHORITY_RESOURCE)
);

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package org.folio.search.service.setter.bibframe.authority;

import org.folio.search.domain.dto.BibframeAuthority;
import org.folio.search.service.setter.FieldProcessor;
import org.springframework.stereotype.Component;

@Component
public class BibframeAuthorityLabelProcessor implements FieldProcessor<BibframeAuthority, String> {

@Override
public String getFieldValue(BibframeAuthority bibframe) {
return bibframe.getLabel();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package org.folio.search.service.setter.bibframe.authority;

import static java.util.stream.Collectors.toCollection;
import static org.folio.search.domain.dto.BibframeAuthorityIdentifiersInner.TypeEnum.LCCN;

import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import lombok.RequiredArgsConstructor;
import org.folio.search.domain.dto.BibframeAuthority;
import org.folio.search.domain.dto.BibframeAuthorityIdentifiersInner;
import org.folio.search.service.lccn.LccnNormalizer;
import org.folio.search.service.setter.FieldProcessor;
import org.springframework.stereotype.Component;

@Component
@RequiredArgsConstructor
public class BibframeAuthorityLccnProcessor implements FieldProcessor<BibframeAuthority, Set<String>> {

private final LccnNormalizer lccnNormalizer;

@Override
public Set<String> getFieldValue(BibframeAuthority bibframe) {
return Optional.of(bibframe)
.map(BibframeAuthority::getIdentifiers)
.orElseGet(Collections::emptyList)
.stream()
.filter(i -> LCCN.equals(i.getType()))
.map(BibframeAuthorityIdentifiersInner::getValue)
.filter(Objects::nonNull)
.map(lccnNormalizer)
.flatMap(Optional::stream)
.collect(toCollection(LinkedHashSet::new));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package org.folio.search.service.setter.bibframe.authority;

import org.folio.search.domain.dto.BibframeAuthority;
import org.folio.search.service.setter.FieldProcessor;
import org.springframework.stereotype.Component;

@Component
public class BibframeAuthorityTypeProcessor implements FieldProcessor<BibframeAuthority, String> {

@Override
public String getFieldValue(BibframeAuthority bibframe) {
var type = bibframe.getType();
if (type == null) {
return null;
}
return type.toString();
}
}
1 change: 1 addition & 0 deletions src/main/java/org/folio/search/utils/SearchUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public class SearchUtils {
public static final String LOCATION_RESOURCE = "location";
public static final String CLASSIFICATION_TYPE_RESOURCE = "classification-type";
public static final String BIBFRAME_RESOURCE = "bibframe";
public static final String BIBFRAME_AUTHORITY_RESOURCE = "bibframe-authority";
public static final String CAMPUS_RESOURCE = "campus";

public static final String ID_FIELD = "id";
Expand Down
2 changes: 1 addition & 1 deletion src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ folio:
group-id: ${folio.environment}-mod-search-location-type-group
bibframe:
concurrency: ${KAFKA_BIBFRAME_CONCURRENCY:1}
topic-pattern: (${folio.environment}\.)(.*\.)search\.bibframe
topic-pattern: (${folio.environment}\.)(.*\.)search\.(bibframe|bibframe-authorities)
group-id: ${folio.environment}-mod-search-bibframe-group
okapiUrl: ${okapi.url}
logging:
Expand Down
68 changes: 68 additions & 0 deletions src/main/resources/elasticsearch/index/bibframe-authority.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
{
"index": {
"number_of_shards": 4,
"number_of_replicas": 2,
"refresh_interval": "1s",
"codec": "best_compression",
"mapping.total_fields.limit": 1000
},
"analysis": {
"filter": {
"folio_word_delimiter_graph": {
"type": "word_delimiter_graph",
"catenate_words": true
}
},
"normalizer": {
"keyword_lowercase": {
"filter": [
"lowercase",
"trim"
],
"type": "custom"
},
"keyword_uppercase": {
"filter": [
"uppercase",
"trim"
],
"type": "custom"
},
"keyword_trimmed": {
"filter": [
"trim"
],
"type": "custom"
}
},
"analyzer": {
"source_analyzer": {
"tokenizer": "icu_tokenizer",
"filter": [
"folio_word_delimiter_graph",
"icu_folding"
],
"char_filter": [
"and_char_replacement"
],
"type": "custom"
},
"whitespace_lowercase_analyzer": {
"tokenizer": "whitespace",
"filter": [
"lowercase",
"icu_folding"
],
"type": "custom"
}
},
"tokenizers": { },
"char_filter": {
"and_char_replacement": {
"type": "pattern_replace",
"pattern": " & ",
"replacement": " and "
}
}
}
}
44 changes: 44 additions & 0 deletions src/main/resources/model/bibframe_authority.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"name": "bibframe-authority",
"eventBodyJavaClass": "org.folio.search.domain.dto.BibframeAuthority",
"fields": {
"id": {
"index": "keyword"
},
"label": {
"index": "multilang"
},
"type": {
"index": "keyword"
},
"identifiers": {
"type": "object",
"properties": {
"value": {
"index": "whitespace"
},
"type": {
"index": "whitespace"
}
}
}
},
"searchFields": {
"label": {
"type": "search",
"index": "multilang",
"processor": "bibframeAuthorityLabelProcessor"
},
"type": {
"type": "search",
"index": "keyword",
"processor": "bibframeAuthorityTypeProcessor"
},
"lccn": {
"type": "search",
"index": "keyword",
"processor": "bibframeAuthorityLccnProcessor"
}
},
"indexMappings": { }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
value:
searchQuery: "query string"
content:
- id: "1"
label: "Label Value"
type: "Person"
identifiers:
- value: "sh85121033"
type: "LCCN"
pageNumber: 0
totalPages: 3
totalRecords: 27
3 changes: 3 additions & 0 deletions src/main/resources/swagger.api/mod-search.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@ paths:
/search/bibframe:
$ref: 'paths/search-bibframe/search-bibframe.yaml'

/search/bibframe/authorities:
$ref: 'paths/search-bibframe/search-bibframe-authorities.yaml'

/browse/call-numbers/instances:
$ref: 'paths/browse-call-numbers/browse-call-numbers-instances.yaml'

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
get:
operationId: searchBibframeAuthorities
summary: Search Bibframe Authorities
description: Get a list of bibframe authorities records for CQL query
tags:
- search
parameters:
- $ref: '../../parameters/x-okapi-tenant-header.yaml'
- $ref: '../../parameters/cql-query.yaml'
- $ref: '../../parameters/bibframe-limit-param.yaml'
- $ref: '../../parameters/offset-param.yaml'
responses:
'200':
description: 'Bibframe authorities search result'
content:
application/json:
schema:
$ref: '../../schemas/response/bibframeSearchAuthorityResult.yaml'
examples:
searchResult:
$ref: '../../examples/result/bibframeSearchAuthorityResult.yaml'
'400':
$ref: '../../responses/badRequestResponse.yaml'
'500':
$ref: '../../responses/internalServerErrorResponse.yaml'
Loading

0 comments on commit cf5d149

Please sign in to comment.