From cbbb31f928bdfa93d39969f2ad6327f42bcdde0d Mon Sep 17 00:00:00 2001 From: Brian Lowe Date: Mon, 1 Aug 2022 17:44:29 +0300 Subject: [PATCH 01/13] Add select query document modifier with dynamic target field; use locale-specific sort fields when available. --- .../freemarker/IndividualListController.java | 34 +++++-- .../SelectQueryDocumentModifier.java | 16 +-- ...eryDocumentModifierDynamicTargetField.java | 98 +++++++++++++++++++ .../utils/searchengine/SearchQueryUtils.java | 40 +++++++- .../everytime/documentModifierI18nSort.n3 | 15 +++ 5 files changed, 181 insertions(+), 22 deletions(-) create mode 100644 api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java create mode 100644 home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java index 012bb6c767..76bd276256 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java @@ -6,9 +6,11 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; -import edu.cornell.mannlib.vitro.webapp.web.templatemodels.individuallist.ListedIndividualBuilder; +import javax.servlet.annotation.WebServlet; + import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -16,6 +18,7 @@ import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup; +import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ExceptionResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues; @@ -27,8 +30,7 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery; import edu.cornell.mannlib.vitro.webapp.utils.searchengine.SearchQueryUtils; import edu.cornell.mannlib.vitro.webapp.web.templatemodels.individuallist.ListedIndividual; - -import javax.servlet.annotation.WebServlet; +import edu.cornell.mannlib.vitro.webapp.web.templatemodels.individuallist.ListedIndividualBuilder; /** * Generates a list of individuals for display in a template @@ -43,6 +45,7 @@ public class IndividualListController extends FreemarkerHttpServlet { private static final int MAX_PAGES = 40; // must be even private static final String TEMPLATE_DEFAULT = "individualList.ftl"; + private static final String LANGUAGE_FILTER_PROPERTY = "RDFService.languageFilter"; @Override protected ResponseValues processRequest(VitroRequest vreq) { @@ -152,11 +155,17 @@ public static int getPageParameter(VitroRequest request) { return SearchQueryUtils.getPageParameter(request); } - public static IndividualListResults getResultsForVClass(String vclassURI, int page, String alpha, VitroRequest vreq) + public static IndividualListResults getResultsForVClass(String vclassURI, + int page, String alpha, VitroRequest vreq) throws SearchException{ try{ + ConfigurationProperties props = ConfigurationProperties.getBean(vreq); + boolean languageFilter = Boolean.valueOf(props.getProperty( + LANGUAGE_FILTER_PROPERTY, "false")); List classUris = Collections.singletonList(vclassURI); - IndividualListQueryResults results = buildAndExecuteVClassQuery(classUris, alpha, page, INDIVIDUALS_PER_PAGE, vreq.getWebappDaoFactory().getIndividualDao()); + IndividualListQueryResults results = buildAndExecuteVClassQuery( + classUris, alpha, ((languageFilter) ? vreq.getLocale() : null), + page, INDIVIDUALS_PER_PAGE, vreq.getWebappDaoFactory().getIndividualDao()); return getResultsForVClassQuery(results, page, INDIVIDUALS_PER_PAGE, alpha, vreq); } catch (SearchEngineException e) { String msg = "An error occurred retrieving results for vclass query"; @@ -169,9 +178,15 @@ public static IndividualListResults getResultsForVClass(String vclassURI, int pa } } - public static IndividualListResults getResultsForVClassIntersections(List vclassURIs, int page, int pageSize, String alpha, VitroRequest vreq) { + public static IndividualListResults getResultsForVClassIntersections( + List vclassURIs, int page, int pageSize, String alpha, VitroRequest vreq) { try{ - IndividualListQueryResults results = buildAndExecuteVClassQuery(vclassURIs, alpha, page, pageSize, vreq.getWebappDaoFactory().getIndividualDao()); + ConfigurationProperties props = ConfigurationProperties.getBean(vreq); + boolean languageFilter = Boolean.valueOf(props.getProperty( + LANGUAGE_FILTER_PROPERTY, "false")); + IndividualListQueryResults results = buildAndExecuteVClassQuery( + vclassURIs, alpha, ((languageFilter) ? vreq.getLocale() : null), + page, pageSize, vreq.getWebappDaoFactory().getIndividualDao()); return getResultsForVClassQuery(results, page, pageSize, alpha, vreq); } catch(Throwable th) { log.error("Error retrieving individuals corresponding to intersection multiple classes." + vclassURIs.toString(), th); @@ -201,9 +216,10 @@ private static IndividualListResults getResultsForVClassQuery(IndividualListQuer private static IndividualListQueryResults buildAndExecuteVClassQuery( - List vclassURIs, String alpha, int page, int pageSize, IndividualDao indDao) + List vclassURIs, String alpha, Locale locale, int page, + int pageSize, IndividualDao indDao) throws SearchEngineException { - SearchQuery query = SearchQueryUtils.getQuery(vclassURIs, alpha, page, pageSize); + SearchQuery query = SearchQueryUtils.getQuery(vclassURIs, alpha, locale, page, pageSize); IndividualListQueryResults results = IndividualListQueryResults.runQuery(query, indDao); log.debug("Executed search query for " + vclassURIs); if (results.getIndividuals().isEmpty()) { diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifier.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifier.java index d4ead0f4aa..f29f328b9c 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifier.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifier.java @@ -52,25 +52,25 @@ public class SelectQueryDocumentModifier implements DocumentModifier, private static final Log log = LogFactory .getLog(SelectQueryDocumentModifier.class); - private RDFService rdfService; + protected RDFService rdfService; /** A name to be used in logging, to identify this instance. */ - private String label; + protected String label; /** The queries to be executed. There must be at least one. */ - private List queries = new ArrayList<>(); + protected List queries = new ArrayList<>(); /** * The names of the fields where the results of the queries will be stored. * If empty, it is assumed to be ALLTEXT and ALLTEXTUNSTEMMED. */ - private List fieldNames = new ArrayList<>(); + protected List fieldNames = new ArrayList<>(); /** * URIs of the types of individuals to whom these queries apply. If empty, * then the queries apply to all individuals. */ - private Set typeRestrictions = new HashSet<>(); + protected Set typeRestrictions = new HashSet<>(); @Override public void setContextModels(ContextModelAccess models) { @@ -128,7 +128,7 @@ public void modifyDocument(Individual ind, SearchInputDocument doc) { } } - private boolean passesTypeRestrictions(Individual ind) { + protected boolean passesTypeRestrictions(Individual ind) { if (typeRestrictions.isEmpty()) { return true; } else { @@ -141,7 +141,7 @@ private boolean passesTypeRestrictions(Individual ind) { return false; } - private List getTextForQueries(Individual ind) { + protected List getTextForQueries(Individual ind) { List list = new ArrayList<>(); for (String query : queries) { list.addAll(getTextForQuery(query, ind)); @@ -149,7 +149,7 @@ private List getTextForQueries(Individual ind) { return list; } - private List getTextForQuery(String query, Individual ind) { + protected List getTextForQuery(String query, Individual ind) { try { QueryHolder queryHolder = new QueryHolder(query).bindToUri("uri", ind.getURI()); diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java new file mode 100644 index 0000000000..e38f5ac388 --- /dev/null +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java @@ -0,0 +1,98 @@ +/* $This file is distributed under the terms of the license in LICENSE$ */ + +package edu.cornell.mannlib.vitro.webapp.searchindex.documentBuilding; + +import static edu.cornell.mannlib.vitro.webapp.utils.sparqlrunner.SparqlQueryRunner.createSelectQueryContext; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import edu.cornell.mannlib.vitro.webapp.beans.Individual; +import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument; +import edu.cornell.mannlib.vitro.webapp.utils.configuration.ContextModelsUser; +import edu.cornell.mannlib.vitro.webapp.utils.configuration.Property; +import edu.cornell.mannlib.vitro.webapp.utils.sparqlrunner.QueryHolder; +import edu.cornell.mannlib.vitro.webapp.utils.sparqlrunner.StringResultsMapping; + +/** + * A variation on SelectQueryDocument where the target field of the search + * index document is specified in the query results. + * + * Each query should contain a ?uri variable, which will be replaced by the URI + * of the individual. + * + * Each query must return a ?targetField variable specifying the name of the + * search document field to be populated. + * + * All of the other result fields in each row of each query will + * be converted to strings and added to the field specified in ?targetField. + * + */ +public class SelectQueryDocumentModifierDynamicTargetField + extends SelectQueryDocumentModifier + implements DocumentModifier, ContextModelsUser { + private static final Log log = LogFactory + .getLog(SelectQueryDocumentModifierDynamicTargetField.class); + + private static final String TARGET_FIELD_VAR = "targetField"; + + @Override + /** + * Grab the un-flattened query solution mappings and use the field name + * specified in the TARGET_FIELD_VAR variable as the location to store the + * rest of the values. + */ + public void modifyDocument(Individual ind, SearchInputDocument doc) { + if (passesTypeRestrictions(ind)) { + List values = getMappingsForQueries(ind); + for(StringResultsMapping value : values) { + for(Map map : value.getListOfMaps()) { + String targetFieldName = map.get(TARGET_FIELD_VAR); + if(targetFieldName == null) { + log.error(label + " select query must return variable " + + TARGET_FIELD_VAR + " to specify document field" + + " in which to store remaining values"); + } else { + for(String key : map.keySet()) { + if(!TARGET_FIELD_VAR.equals(key)) { + doc.addField(targetFieldName, map.get(key)); + if(log.isDebugEnabled()) { + log.debug("Added field " + targetFieldName + + " value " + map.get(key) + " to " + + ind.getURI()); + } + } + } + } + } + } + } + } + + protected List getMappingsForQueries(Individual ind) { + List list = new ArrayList<>(); + for (String query : queries) { + list.add(getQueryResults(query, ind)); + } + return list; + } + + protected StringResultsMapping getQueryResults(String query, Individual ind) { + try { + QueryHolder queryHolder = new QueryHolder(query).bindToUri("uri", + ind.getURI()); + StringResultsMapping mapping = createSelectQueryContext(rdfService, + queryHolder).execute().toStringFields(); + log.debug(label + " query: '" + query + "' returns " + mapping); + return mapping; + } catch (Throwable t) { + log.error("problem while running query '" + query + "'", t); + return StringResultsMapping.EMPTY; + } + } + +} diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java index a9a03e6e91..17115cb617 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Locale; import java.util.Map; import org.apache.commons.lang3.StringUtils; @@ -150,19 +151,38 @@ public static long getIndividualCount(List vclassUris) { } /** - * builds a query with a type clause for each type in vclassUris, NAME_LOWERCASE filetred by - * alpha, and just the hits for the page for pageSize. + * builds a query with a type clause for each type in vclassUris, + * NAME_LOWERCASE filtered by alpha, and just the hits for the page for pageSize. + * @param locale may be null. If null, default sort field will be used. + * Otherwise, query will be sorted by locale-specific sort field. */ - public static SearchQuery getQuery(List vclassUris, String alpha, int page, int pageSize){ + public static SearchQuery getQuery(List vclassUris, String alpha, + Locale locale, int page, int pageSize){ String queryText = ""; SearchEngine searchEngine = ApplicationUtils.instance().getSearchEngine(); try { queryText = makeMultiClassQuery(vclassUris); - + + String localeSpecificField = null; + + if (locale != null) { + localeSpecificField = getSortFieldNameForLocale(locale); + } + // Add alpha filter if applicable if ( alpha != null && !"".equals(alpha) && alpha.length() == 1) { - queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*"; + if (locale == null) { + queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*"; + } else { + // Retrieve items matching the appropriate alpha char + // on the i18ned field if that field exists. For records + // where the field does not exist, fall back to NAME_LOWERCASE + queryText += "(" + localeSpecificField + ":" + alpha.toLowerCase() + + "* OR (-" + localeSpecificField + ":[* TO *] AND " + + VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*))"; + log.debug("Multiclass query text: " + queryText); + } } SearchQuery query = searchEngine.createQuery(queryText); @@ -172,6 +192,11 @@ public static SearchQuery getQuery(List vclassUris, String alpha, int pa query.setStart( startRow ).setRows( pageSize ); // Need a single-valued field for sorting + // Sort first by sort field for locale; fall back to + // NAME_LOWERCASE_SINGLE_VALUED if not available. + if(locale != null) { + query.addSortField(localeSpecificField, Order.ASC); + } query.addSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, Order.ASC); log.debug("Query is " + query.toString()); @@ -182,6 +207,11 @@ public static SearchQuery getQuery(List vclassUris, String alpha, int pa return searchEngine.createQuery(); } } + + public static String getSortFieldNameForLocale(Locale locale) { + return VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED + "_" + + locale.toString().replace('_', '-') + "_s"; + } public static SearchQuery getRandomQuery(List vclassUris, int page, int pageSize){ String queryText = ""; diff --git a/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 b/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 new file mode 100644 index 0000000000..d3a5a8c55c --- /dev/null +++ b/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 @@ -0,0 +1,15 @@ +@prefix : . +@prefix rdfs: . +@prefix xsd: . + +:documentModifier_multilingual_sort + a , + ; + rdfs:label "multilingual sort document modifier" ; + :hasSelectQuery """ + PREFIX rdfs: + SELECT ?targetField (LCASE(STR(MIN(?labelRaw))) AS ?label) WHERE { + ?uri rdfs:label ?labelRaw + BIND(CONCAT("nameLowercaseSingleValued_", IF(LANG(?labelRaw) = "", "plain", LANG(?labelRaw)), "_s") AS ?targetField) + } GROUP BY ?targetField + """ . From 6beeac7840adc08e0b3ac0334f5af190561db56f Mon Sep 17 00:00:00 2001 From: Brian Lowe Date: Tue, 2 Aug 2022 12:29:56 +0300 Subject: [PATCH 02/13] Add i18nized labels to index for autocomplete --- .../controller/AutocompleteController.java | 10 ++++++++-- .../utils/searchengine/SearchQueryUtils.java | 5 +++++ .../everytime/documentModifierI18nLabel.n3 | 16 ++++++++++++++++ .../everytime/documentModifierI18nSort.n3 | 3 ++- 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/controller/AutocompleteController.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/controller/AutocompleteController.java index ab16e25180..db79eb5f09 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/controller/AutocompleteController.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/controller/AutocompleteController.java @@ -36,6 +36,7 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; +import edu.cornell.mannlib.vitro.webapp.utils.searchengine.SearchQueryUtils; /** * AutocompleteController generates autocomplete content @@ -127,7 +128,10 @@ else if ( parts.length == 1 ) { for (SearchResultDocument doc : docs) { try { String uri = doc.getStringValue(VitroSearchTermNames.URI); - String name = doc.getStringValue(VitroSearchTermNames.NAME_RAW); + String name = doc.getStringValue(SearchQueryUtils.getLabelFieldNameForLocale(vreq.getLocale())); + if (name == null) { + name = doc.getStringValue(VitroSearchTermNames.NAME_RAW); + } //There may be multiple most specific types, sending them all back String mst = doc.getStringValue(VitroSearchTermNames.MOST_SPECIFIC_TYPE_URIS); //Assuming these will get me string values @@ -184,7 +188,9 @@ private SearchQuery getQuery(String queryStr, VitroRequest vreq) { addFilterQuery(query, typeParam, multipleTypesParam); } - query.addFields(VitroSearchTermNames.NAME_RAW, VitroSearchTermNames.URI, VitroSearchTermNames.MOST_SPECIFIC_TYPE_URIS); // fields to retrieve + query.addFields(SearchQueryUtils.getLabelFieldNameForLocale(vreq.getLocale()), + VitroSearchTermNames.NAME_RAW, VitroSearchTermNames.URI, + VitroSearchTermNames.MOST_SPECIFIC_TYPE_URIS); // fields to retrieve // Can't sort on multivalued field, so we sort the results in Java when we get them. // query.addSortField(VitroSearchTermNames.NAME_LOWERCASE, Order.ASC); diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java index 17115cb617..763ab95836 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java @@ -212,6 +212,11 @@ public static String getSortFieldNameForLocale(Locale locale) { return VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED + "_" + locale.toString().replace('_', '-') + "_s"; } + + public static String getLabelFieldNameForLocale(Locale locale) { + return VitroSearchTermNames.NAME_RAW + "SingleValued_" + + locale.toString().replace('_', '-') + "_s"; + } public static SearchQuery getRandomQuery(List vclassUris, int page, int pageSize){ String queryText = ""; diff --git a/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 b/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 new file mode 100644 index 0000000000..62ce2380f4 --- /dev/null +++ b/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 @@ -0,0 +1,16 @@ +@prefix : . +@prefix rdfs: . +@prefix xsd: . + +:documentModifier_multilingual_label + a , + ; + rdfs:label "multilingual label document modifier" ; + :hasSelectQuery """ + PREFIX rdfs: + SELECT ?targetField (LCASE(STR(MIN(?labelRaw))) AS ?label) WHERE { + ?uri rdfs:label ?labelRaw + FILTER("" != LANG(?labelRaw)) + BIND(CONCAT("nameRawSingleValued_", LANG(?labelRaw), "_s") AS ?targetField) + } GROUP BY ?targetField + """ . diff --git a/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 b/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 index d3a5a8c55c..6ec2ce5adf 100644 --- a/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 +++ b/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 @@ -10,6 +10,7 @@ PREFIX rdfs: SELECT ?targetField (LCASE(STR(MIN(?labelRaw))) AS ?label) WHERE { ?uri rdfs:label ?labelRaw - BIND(CONCAT("nameLowercaseSingleValued_", IF(LANG(?labelRaw) = "", "plain", LANG(?labelRaw)), "_s") AS ?targetField) + FILTER("" != LANG(?labelRaw)) + BIND(CONCAT("nameLowercaseSingleValued_", LANG(?labelRaw), "_s") AS ?targetField) } GROUP BY ?targetField """ . From 4faad6aff74075811c9b0851d848c34d03ab5094 Mon Sep 17 00:00:00 2001 From: Brian Lowe Date: Tue, 2 Aug 2022 12:33:51 +0300 Subject: [PATCH 03/13] Remove lowercasing from label query --- .../rdf/display/everytime/documentModifierI18nLabel.n3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 b/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 index 62ce2380f4..f7af89975b 100644 --- a/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 +++ b/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 @@ -8,7 +8,7 @@ rdfs:label "multilingual label document modifier" ; :hasSelectQuery """ PREFIX rdfs: - SELECT ?targetField (LCASE(STR(MIN(?labelRaw))) AS ?label) WHERE { + SELECT ?targetField (STR(MIN(?labelRaw)) AS ?label) WHERE { ?uri rdfs:label ?labelRaw FILTER("" != LANG(?labelRaw)) BIND(CONCAT("nameRawSingleValued_", LANG(?labelRaw), "_s") AS ?targetField) From 5612efd27ee60350cb206c2c4c453e0adec30215 Mon Sep 17 00:00:00 2001 From: Georgy Litvinov Date: Wed, 3 Aug 2022 19:18:11 +0200 Subject: [PATCH 04/13] Improved document modifier for multilingual field with defined suffix name --- .../webapp/search/VitroSearchTermNames.java | 6 + .../SelectQueryDocumentModifier.java | 4 +- ...eryDocumentModifierDynamicTargetField.java | 150 ++++++++++-------- .../configuration/ConfigurationReader.java | 13 ++ .../utils/configuration/WrappedInstance.java | 10 ++ .../utils/searchengine/SearchQueryUtils.java | 6 +- .../everytime/documentModifierI18nLabel.n3 | 9 +- .../everytime/documentModifierI18nSort.n3 | 9 +- 8 files changed, 122 insertions(+), 85 deletions(-) create mode 100644 api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/configuration/ConfigurationReader.java diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/VitroSearchTermNames.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/VitroSearchTermNames.java index 1469e23cfe..73c469f86b 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/VitroSearchTermNames.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/VitroSearchTermNames.java @@ -70,5 +70,11 @@ public class VitroSearchTermNames { /** Source institution name */ public static final String SITE_NAME = "siteName"; + + /** Multilingual sort field suffix */ + public static final String SORT_SUFFIX = "_label_sort"; + + /** Multilingual label field suffix */ + public static final String LABEL_DISPLAY_SUFFIX = "_label_display"; } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifier.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifier.java index f29f328b9c..d9261a232e 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifier.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifier.java @@ -141,7 +141,7 @@ protected boolean passesTypeRestrictions(Individual ind) { return false; } - protected List getTextForQueries(Individual ind) { + private List getTextForQueries(Individual ind) { List list = new ArrayList<>(); for (String query : queries) { list.addAll(getTextForQuery(query, ind)); @@ -149,7 +149,7 @@ protected List getTextForQueries(Individual ind) { return list; } - protected List getTextForQuery(String query, Individual ind) { + private List getTextForQuery(String query, Individual ind) { try { QueryHolder queryHolder = new QueryHolder(query).bindToUri("uri", ind.getURI()); diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java index e38f5ac388..cd4179bec5 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java @@ -3,96 +3,108 @@ package edu.cornell.mannlib.vitro.webapp.searchindex.documentBuilding; import static edu.cornell.mannlib.vitro.webapp.utils.sparqlrunner.SparqlQueryRunner.createSelectQueryContext; - +import static edu.cornell.mannlib.vitro.webapp.i18n.selection.LocaleSelectionSetup.PROPERTY_SELECTABLE_LOCALES; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import edu.cornell.mannlib.vitro.webapp.beans.Individual; +import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument; +import edu.cornell.mannlib.vitro.webapp.rdfservice.filter.LanguageFilteringRDFService; +import edu.cornell.mannlib.vitro.webapp.utils.configuration.ConfigurationReader; import edu.cornell.mannlib.vitro.webapp.utils.configuration.ContextModelsUser; import edu.cornell.mannlib.vitro.webapp.utils.configuration.Property; import edu.cornell.mannlib.vitro.webapp.utils.sparqlrunner.QueryHolder; -import edu.cornell.mannlib.vitro.webapp.utils.sparqlrunner.StringResultsMapping; /** - * A variation on SelectQueryDocument where the target field of the search - * index document is specified in the query results. - * + * A variation on SelectQueryDocument where the suffix of target field is defined. + * Multiple queries are performed for each of locales configured in runtime.properties + * + * Target field names are composed of locale + fieldSuffix. + * * Each query should contain a ?uri variable, which will be replaced by the URI * of the individual. * - * Each query must return a ?targetField variable specifying the name of the - * search document field to be populated. - * - * All of the other result fields in each row of each query will - * be converted to strings and added to the field specified in ?targetField. + * All of the other result fields in each row of each query will be converted to + * strings and added to the field. * */ -public class SelectQueryDocumentModifierDynamicTargetField - extends SelectQueryDocumentModifier - implements DocumentModifier, ContextModelsUser { - private static final Log log = LogFactory - .getLog(SelectQueryDocumentModifierDynamicTargetField.class); - - private static final String TARGET_FIELD_VAR = "targetField"; +public class SelectQueryDocumentModifierDynamicTargetField extends SelectQueryDocumentModifier + implements DocumentModifier, ContextModelsUser, ConfigurationReader { + private static final Log log = LogFactory.getLog(SelectQueryDocumentModifierDynamicTargetField.class); + + private String fieldSuffix = ""; + + private ArrayList locales = new ArrayList<>(); + + @Property(uri = "http://vitro.mannlib.cornell.edu/ns/vitro/ApplicationSetup#hasTargetSuffix") + public void setTargetSuffix(String fieldSuffix) { + this.fieldSuffix = fieldSuffix; + } + + @Override + public void modifyDocument(Individual ind, SearchInputDocument doc) { + if (passesTypeRestrictions(ind) && StringUtils.isNotBlank(fieldSuffix)) { + List>> maps = getTextForQueries(ind); + for (Map> map : maps) { + for (String locale : map.keySet()) { + List values = map.get(locale); + String fieldName = locale + fieldSuffix; + doc.addField(fieldName, values); + } + } + } + } + + protected List>> getTextForQueries(Individual ind) { + List>> list = new ArrayList<>(); + for (String query : queries) { + list.add(getTextForQuery(query, ind)); + } + return list; + } - @Override - /** - * Grab the un-flattened query solution mappings and use the field name - * specified in the TARGET_FIELD_VAR variable as the location to store the - * rest of the values. - */ - public void modifyDocument(Individual ind, SearchInputDocument doc) { - if (passesTypeRestrictions(ind)) { - List values = getMappingsForQueries(ind); - for(StringResultsMapping value : values) { - for(Map map : value.getListOfMaps()) { - String targetFieldName = map.get(TARGET_FIELD_VAR); - if(targetFieldName == null) { - log.error(label + " select query must return variable " - + TARGET_FIELD_VAR + " to specify document field" - + " in which to store remaining values"); - } else { - for(String key : map.keySet()) { - if(!TARGET_FIELD_VAR.equals(key)) { - doc.addField(targetFieldName, map.get(key)); - if(log.isDebugEnabled()) { - log.debug("Added field " + targetFieldName - + " value " + map.get(key) + " to " - + ind.getURI()); - } - } - } - } - } - } - } - } + protected Map> getTextForQuery(String query, Individual ind) { + try { + QueryHolder queryHolder = new QueryHolder(query).bindToUri("uri", ind.getURI()); + Map> mapLocaleToFields = new HashMap<>(); + for (String locale : locales) { + LanguageFilteringRDFService lfrs = new LanguageFilteringRDFService(rdfService, + Collections.singletonList(locale)); + List list = createSelectQueryContext(lfrs, queryHolder).execute().toStringFields().flatten(); + mapLocaleToFields.put(locale, list); + log.debug(label + " for locale " + locale + " - query: '" + query + "' returns " + list); + } + return mapLocaleToFields; + } catch (Throwable t) { + log.error("problem while running query '" + query + "'", t); + return Collections.emptyMap(); + } + } - protected List getMappingsForQueries(Individual ind) { - List list = new ArrayList<>(); - for (String query : queries) { - list.add(getQueryResults(query, ind)); - } - return list; - } + @Override + public void setConfigurationProperties(ConfigurationProperties config) { + String property = config.getProperty(PROPERTY_SELECTABLE_LOCALES); + if (!StringUtils.isBlank(property)) { + String[] values = property.trim().split("\\s*,\\s*"); + for (String value : values) { + addLocale(value); + } + } + } - protected StringResultsMapping getQueryResults(String query, Individual ind) { - try { - QueryHolder queryHolder = new QueryHolder(query).bindToUri("uri", - ind.getURI()); - StringResultsMapping mapping = createSelectQueryContext(rdfService, - queryHolder).execute().toStringFields(); - log.debug(label + " query: '" + query + "' returns " + mapping); - return mapping; - } catch (Throwable t) { - log.error("problem while running query '" + query + "'", t); - return StringResultsMapping.EMPTY; - } - } + private void addLocale(String localeString) { + if (StringUtils.isBlank(localeString)) { + return; + } + locales.add(localeString); + } } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/configuration/ConfigurationReader.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/configuration/ConfigurationReader.java new file mode 100644 index 0000000000..29d01996d5 --- /dev/null +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/configuration/ConfigurationReader.java @@ -0,0 +1,13 @@ +/* $This file is distributed under the terms of the license in LICENSE$ */ + +package edu.cornell.mannlib.vitro.webapp.utils.configuration; + +import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; + +/** + * When the ConfigurationBeanLoader creates an instance of this class, it will + * call this method, supplying ConfigurationProperties. + */ +public interface ConfigurationReader { + void setConfigurationProperties(ConfigurationProperties properties); +} diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/configuration/WrappedInstance.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/configuration/WrappedInstance.java index 5133bf2a4c..9e889426ce 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/configuration/WrappedInstance.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/configuration/WrappedInstance.java @@ -12,6 +12,7 @@ import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; +import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess; import edu.cornell.mannlib.vitro.webapp.utils.configuration.PropertyType.PropertyMethod; import edu.cornell.mannlib.vitro.webapp.utils.configuration.PropertyType.PropertyStatement; @@ -62,6 +63,15 @@ public void satisfyInterfaces(ServletContext ctx, HttpServletRequest req) rmu.setRequestModels(ModelAccess.on(req)); } } + if (instance instanceof ConfigurationReader) { + if (ctx == null) { + throw new ResourceUnavailableException("Cannot satisfy " + + "ConfigurationReader interface: context not available."); + } else { + ConfigurationReader cr = (ConfigurationReader) instance; + cr.setConfigurationProperties(ConfigurationProperties.getBean(ctx)); + } + } } /** diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java index 763ab95836..3b408ac949 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java @@ -209,13 +209,11 @@ public static SearchQuery getQuery(List vclassUris, String alpha, } public static String getSortFieldNameForLocale(Locale locale) { - return VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED + "_" - + locale.toString().replace('_', '-') + "_s"; + return locale.toString() + VitroSearchTermNames.SORT_SUFFIX; } public static String getLabelFieldNameForLocale(Locale locale) { - return VitroSearchTermNames.NAME_RAW + "SingleValued_" - + locale.toString().replace('_', '-') + "_s"; + return locale.toString() + VitroSearchTermNames.LABEL_DISPLAY_SUFFIX; } public static SearchQuery getRandomQuery(List vclassUris, int page, int pageSize){ diff --git a/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 b/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 index f7af89975b..7f37b4d881 100644 --- a/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 +++ b/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 @@ -6,11 +6,10 @@ a , ; rdfs:label "multilingual label document modifier" ; + :hasTargetSuffix "_label_display" ; :hasSelectQuery """ PREFIX rdfs: - SELECT ?targetField (STR(MIN(?labelRaw)) AS ?label) WHERE { - ?uri rdfs:label ?labelRaw - FILTER("" != LANG(?labelRaw)) - BIND(CONCAT("nameRawSingleValued_", LANG(?labelRaw), "_s") AS ?targetField) - } GROUP BY ?targetField + SELECT ( MIN(?label) as ?singleLabel ) WHERE { + ?uri rdfs:label ?label . + } """ . diff --git a/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 b/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 index 6ec2ce5adf..5560e01742 100644 --- a/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 +++ b/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 @@ -6,11 +6,10 @@ a , ; rdfs:label "multilingual sort document modifier" ; + :hasTargetSuffix "_label_sort" ; :hasSelectQuery """ PREFIX rdfs: - SELECT ?targetField (LCASE(STR(MIN(?labelRaw))) AS ?label) WHERE { - ?uri rdfs:label ?labelRaw - FILTER("" != LANG(?labelRaw)) - BIND(CONCAT("nameLowercaseSingleValued_", LANG(?labelRaw), "_s") AS ?targetField) - } GROUP BY ?targetField + SELECT ( MIN(?label) as ?singleLabel ) WHERE { + ?uri rdfs:label ?label . + } """ . From 3546931330273414e04c3bebb4e2d2bd4067f216 Mon Sep 17 00:00:00 2001 From: Georgy Litvinov Date: Thu, 4 Aug 2022 17:11:19 +0200 Subject: [PATCH 05/13] Improved document modifier for multilingual field with defined suffix name --- .../solr/SolrFieldInitializer.java | 72 +++++++++++++++++++ .../searchengine/solr/SolrSearchEngine.java | 4 +- 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrFieldInitializer.java diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrFieldInitializer.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrFieldInitializer.java new file mode 100644 index 0000000000..f9cbf3c706 --- /dev/null +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrFieldInitializer.java @@ -0,0 +1,72 @@ +package edu.cornell.mannlib.vitro.webapp.searchengine.solr; + +import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.LABEL_DISPLAY_SUFFIX; +import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.SORT_SUFFIX; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; +import org.apache.solr.client.solrj.request.schema.SchemaRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.schema.SchemaResponse; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.SimpleOrderedMap; + +public class SolrFieldInitializer { + + static void initializeFields(SolrClient queryEngine, ConcurrentUpdateSolrClient updateEngine) throws Exception { + Set fieldSuffixes = new HashSet<>(Arrays.asList(SORT_SUFFIX, LABEL_DISPLAY_SUFFIX)); + excludeMatchedFields(fieldSuffixes, queryEngine, "dynamicFields"); + excludeMatchedFields(fieldSuffixes, queryEngine, "fields"); + createMissingFields(fieldSuffixes, updateEngine); + } + + private static void createMissingFields(Set fieldSuffixes, ConcurrentUpdateSolrClient updateEngine) + throws Exception { + for (String suffix : fieldSuffixes) { + Map fieldAttributes = getFieldAttributes(suffix); + SchemaRequest.AddDynamicField request = new SchemaRequest.AddDynamicField(fieldAttributes); + SchemaResponse.UpdateResponse response = request.process(updateEngine); + if (response.getStatus() != 0) { + throw new Exception("Creation of missing solr field '*" + suffix + "' failed"); + } + } + } + + private static Map getFieldAttributes(String suffix) { + Map fieldAttributes = new HashMap(); + fieldAttributes.put("type", "string"); + fieldAttributes.put("stored", "true"); + fieldAttributes.put("indexed", "true"); + fieldAttributes.put("name", "*" + suffix); + return fieldAttributes; + } + + private static void excludeMatchedFields(Set fieldSuffixes, SolrClient queryEngine, String fieldType) + throws Exception { + SolrQuery query = new SolrQuery(); + query.add(CommonParams.QT, "/schema/" + fieldType.toLowerCase()); + QueryResponse response = queryEngine.query(query); + ArrayList fieldList = (ArrayList) response.getResponse().get(fieldType); + if (fieldList == null) { + return; + } + Set it = new HashSet<>(fieldSuffixes); + for (String target : it) { + for (SimpleOrderedMap field : fieldList) { + String fieldName = (String) field.get("name"); + if (fieldName.endsWith(target)) { + fieldSuffixes.remove(target); + } + } + } + } + +} diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java index fbea3bb956..e7ec7fd569 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java @@ -76,7 +76,9 @@ public void startup(Application application, ComponentStartupStatus css) { // no apparent 7.4.0 analogy to `setPollQueueTime(25)` updateEngine = updateBuilder.build(); - + + SolrFieldInitializer.initializeFields(queryEngine, updateEngine); + css.info("Set up the Solr search engine; URL = '" + solrServerUrlString + "'."); } catch (Exception e) { css.fatal("Could not set up the Solr search engine", e); From ebeef0dd3396346919ab0bc63dd6af0dd5e5ccf4 Mon Sep 17 00:00:00 2001 From: Georgy Litvinov Date: Fri, 5 Aug 2022 11:39:24 +0200 Subject: [PATCH 06/13] refact: reverted access modifier changes --- .../documentBuilding/SelectQueryDocumentModifier.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifier.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifier.java index d9261a232e..ff63a1c88d 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifier.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifier.java @@ -64,13 +64,13 @@ public class SelectQueryDocumentModifier implements DocumentModifier, * The names of the fields where the results of the queries will be stored. * If empty, it is assumed to be ALLTEXT and ALLTEXTUNSTEMMED. */ - protected List fieldNames = new ArrayList<>(); + private List fieldNames = new ArrayList<>(); /** * URIs of the types of individuals to whom these queries apply. If empty, * then the queries apply to all individuals. */ - protected Set typeRestrictions = new HashSet<>(); + private Set typeRestrictions = new HashSet<>(); @Override public void setContextModels(ContextModelAccess models) { From f75da27c3e955e7396126c78ed1afaa94512b782 Mon Sep 17 00:00:00 2001 From: Georgy Litvinov Date: Fri, 5 Aug 2022 11:45:59 +0200 Subject: [PATCH 07/13] Lowercase label in documentModifierI18nSort in case old solr schema is used which doesn't have lowercase filter --- .../resources/rdf/display/everytime/documentModifierI18nSort.n3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 b/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 index 5560e01742..d1dddf58f6 100644 --- a/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 +++ b/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 @@ -9,7 +9,7 @@ :hasTargetSuffix "_label_sort" ; :hasSelectQuery """ PREFIX rdfs: - SELECT ( MIN(?label) as ?singleLabel ) WHERE { + SELECT ( LCASE(STR(MIN(?label))) as ?singleLabel ) WHERE { ?uri rdfs:label ?label . } """ . From 669bd1847ce17bab5591d7a8fc5d1ecd38582943 Mon Sep 17 00:00:00 2001 From: Georgy Litvinov Date: Thu, 11 Aug 2022 15:32:34 +0200 Subject: [PATCH 08/13] fix: fixed queries and locale names --- .../SelectQueryDocumentModifierDynamicTargetField.java | 3 ++- .../vitro/webapp/utils/searchengine/SearchQueryUtils.java | 4 ++-- .../rdf/display/everytime/documentModifierI18nLabel.n3 | 5 +++-- .../rdf/display/everytime/documentModifierI18nSort.n3 | 7 ++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java index cd4179bec5..3d74a1df44 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java @@ -95,7 +95,8 @@ public void setConfigurationProperties(ConfigurationProperties config) { if (!StringUtils.isBlank(property)) { String[] values = property.trim().split("\\s*,\\s*"); for (String value : values) { - addLocale(value); + String locale = value.replace("_", "-"); + addLocale(locale); } } } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java index 3b408ac949..7ede76f62b 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java @@ -209,11 +209,11 @@ public static SearchQuery getQuery(List vclassUris, String alpha, } public static String getSortFieldNameForLocale(Locale locale) { - return locale.toString() + VitroSearchTermNames.SORT_SUFFIX; + return locale.toString().replace('_', '-') + VitroSearchTermNames.SORT_SUFFIX; } public static String getLabelFieldNameForLocale(Locale locale) { - return locale.toString() + VitroSearchTermNames.LABEL_DISPLAY_SUFFIX; + return locale.toString().replace('_', '-') + VitroSearchTermNames.LABEL_DISPLAY_SUFFIX; } public static SearchQuery getRandomQuery(List vclassUris, int page, int pageSize){ diff --git a/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 b/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 index 7f37b4d881..86277c58c5 100644 --- a/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 +++ b/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 @@ -9,7 +9,8 @@ :hasTargetSuffix "_label_display" ; :hasSelectQuery """ PREFIX rdfs: - SELECT ( MIN(?label) as ?singleLabel ) WHERE { + SELECT (MIN(?label) AS ?singleLabel ) WHERE { ?uri rdfs:label ?label . - } + BIND (LANG(?label) as ?lang ) + } GROUP BY ?lang ORDER BY ?lang """ . diff --git a/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 b/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 index d1dddf58f6..6719c0efa4 100644 --- a/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 +++ b/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 @@ -9,7 +9,8 @@ :hasTargetSuffix "_label_sort" ; :hasSelectQuery """ PREFIX rdfs: - SELECT ( LCASE(STR(MIN(?label))) as ?singleLabel ) WHERE { - ?uri rdfs:label ?label . - } + SELECT ( LCASE(MIN(?label)) AS ?singleLabel ) WHERE { + ?uri rdfs:label ?label . + BIND (LANG(?label) as ?lang ) + } GROUP BY ?lang ORDER BY ?lang """ . From f32de73f67755883f6ff8329c98d8c927eb5c77f Mon Sep 17 00:00:00 2001 From: Georgy Litvinov Date: Thu, 11 Aug 2022 18:57:01 +0200 Subject: [PATCH 09/13] fix: renamed new document modifier --- ...argetField.java => SelectQueryI18nDocumentModifier.java} | 6 +++--- .../rdf/display/everytime/documentModifierI18nLabel.n3 | 2 +- .../rdf/display/everytime/documentModifierI18nSort.n3 | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/{SelectQueryDocumentModifierDynamicTargetField.java => SelectQueryI18nDocumentModifier.java} (93%) diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryI18nDocumentModifier.java similarity index 93% rename from api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java rename to api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryI18nDocumentModifier.java index 3d74a1df44..89d81ebcfb 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryDocumentModifierDynamicTargetField.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SelectQueryI18nDocumentModifier.java @@ -24,7 +24,7 @@ import edu.cornell.mannlib.vitro.webapp.utils.sparqlrunner.QueryHolder; /** - * A variation on SelectQueryDocument where the suffix of target field is defined. + * A variation on SelectQueryDocumentModifier where the suffix of target field is defined. * Multiple queries are performed for each of locales configured in runtime.properties * * Target field names are composed of locale + fieldSuffix. @@ -36,9 +36,9 @@ * strings and added to the field. * */ -public class SelectQueryDocumentModifierDynamicTargetField extends SelectQueryDocumentModifier +public class SelectQueryI18nDocumentModifier extends SelectQueryDocumentModifier implements DocumentModifier, ContextModelsUser, ConfigurationReader { - private static final Log log = LogFactory.getLog(SelectQueryDocumentModifierDynamicTargetField.class); + private static final Log log = LogFactory.getLog(SelectQueryI18nDocumentModifier.class); private String fieldSuffix = ""; diff --git a/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 b/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 index 86277c58c5..88e884b0ab 100644 --- a/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 +++ b/home/src/main/resources/rdf/display/everytime/documentModifierI18nLabel.n3 @@ -3,7 +3,7 @@ @prefix xsd: . :documentModifier_multilingual_label - a , + a , ; rdfs:label "multilingual label document modifier" ; :hasTargetSuffix "_label_display" ; diff --git a/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 b/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 index 6719c0efa4..9c3002ff1f 100644 --- a/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 +++ b/home/src/main/resources/rdf/display/everytime/documentModifierI18nSort.n3 @@ -3,7 +3,7 @@ @prefix xsd: . :documentModifier_multilingual_sort - a , + a , ; rdfs:label "multilingual sort document modifier" ; :hasTargetSuffix "_label_sort" ; From 2d94402726aa78e8d0356773f0e81dc086c8ce1b Mon Sep 17 00:00:00 2001 From: Georgy Litvinov Date: Fri, 12 Aug 2022 11:07:52 +0200 Subject: [PATCH 10/13] fix: use linkedHashMap to retain map sort fields order --- .../vitro/webapp/searchengine/base/BaseSearchQuery.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java index 4e32560639..47eff17a68 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java @@ -7,6 +7,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; @@ -21,7 +22,7 @@ public class BaseSearchQuery implements SearchQuery { private int rows = -1; private final Set fieldsToReturn = new HashSet<>(); - private final Map sortFields = new HashMap<>(); + private final Map sortFields = new LinkedHashMap <>(); private final Set filters = new HashSet<>(); private final Set facetFields = new HashSet<>(); From 7bbb7c1e7c8fca21692462cfbbdca34349d02954 Mon Sep 17 00:00:00 2001 From: Georgy Litvinov Date: Wed, 17 Aug 2022 12:56:30 +0200 Subject: [PATCH 11/13] refact: extracted buildAndExecuteVClassQuery(List classUris, int page, int pageSize, String alpha, VitroRequest vreq) --- .../freemarker/IndividualListController.java | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java index 76bd276256..3b74a2208b 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java @@ -159,14 +159,8 @@ public static IndividualListResults getResultsForVClass(String vclassURI, int page, String alpha, VitroRequest vreq) throws SearchException{ try{ - ConfigurationProperties props = ConfigurationProperties.getBean(vreq); - boolean languageFilter = Boolean.valueOf(props.getProperty( - LANGUAGE_FILTER_PROPERTY, "false")); - List classUris = Collections.singletonList(vclassURI); - IndividualListQueryResults results = buildAndExecuteVClassQuery( - classUris, alpha, ((languageFilter) ? vreq.getLocale() : null), - page, INDIVIDUALS_PER_PAGE, vreq.getWebappDaoFactory().getIndividualDao()); - return getResultsForVClassQuery(results, page, INDIVIDUALS_PER_PAGE, alpha, vreq); + List classUris = Collections.singletonList(vclassURI); + return buildAndExecuteVClassQuery(classUris, page, INDIVIDUALS_PER_PAGE, alpha, vreq); } catch (SearchEngineException e) { String msg = "An error occurred retrieving results for vclass query"; log.error(msg, e); @@ -179,21 +173,26 @@ public static IndividualListResults getResultsForVClass(String vclassURI, } public static IndividualListResults getResultsForVClassIntersections( - List vclassURIs, int page, int pageSize, String alpha, VitroRequest vreq) { + List classUris, int page, int pageSize, String alpha, VitroRequest vreq) { try{ - ConfigurationProperties props = ConfigurationProperties.getBean(vreq); - boolean languageFilter = Boolean.valueOf(props.getProperty( - LANGUAGE_FILTER_PROPERTY, "false")); - IndividualListQueryResults results = buildAndExecuteVClassQuery( - vclassURIs, alpha, ((languageFilter) ? vreq.getLocale() : null), - page, pageSize, vreq.getWebappDaoFactory().getIndividualDao()); - return getResultsForVClassQuery(results, page, pageSize, alpha, vreq); + return buildAndExecuteVClassQuery(classUris, page, pageSize, alpha, vreq); } catch(Throwable th) { - log.error("Error retrieving individuals corresponding to intersection multiple classes." + vclassURIs.toString(), th); + log.error("Error retrieving individuals corresponding to intersection multiple classes." + classUris.toString(), th); return IndividualListResults.EMPTY; } } + private static IndividualListResults buildAndExecuteVClassQuery(List classUris, int page, int pageSize, + String alpha, VitroRequest vreq) throws SearchEngineException { + ConfigurationProperties props = ConfigurationProperties.getBean(vreq); + boolean languageFilter = Boolean.valueOf(props.getProperty(LANGUAGE_FILTER_PROPERTY, "false")); + IndividualListQueryResults results = buildAndExecuteVClassQuery(classUris, alpha, + ((languageFilter) ? vreq.getLocale() : null), page, pageSize, + vreq.getWebappDaoFactory().getIndividualDao()); + IndividualListResults indListResults = getResultsForVClassQuery(results, page, pageSize, alpha, vreq); + return indListResults; + } + public static IndividualListResults getRandomResultsForVClass(String vclassURI, int page, int pageSize, VitroRequest vreq) { try{ List classUris = Collections.singletonList(vclassURI); From 7a9673893ec4339610b2ace75b8ed1de82faba8a Mon Sep 17 00:00:00 2001 From: Georgy Litvinov Date: Wed, 17 Aug 2022 12:58:11 +0200 Subject: [PATCH 12/13] fix: removed unused import --- .../mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java index 47eff17a68..83f6b5b343 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java @@ -5,7 +5,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; From 52d70b2c09793aa5f7b4e1458b84484dfae200ee Mon Sep 17 00:00:00 2001 From: Georgy Litvinov Date: Wed, 17 Aug 2022 13:00:50 +0200 Subject: [PATCH 13/13] fix: constant name aligned with other suffix --- .../mannlib/vitro/webapp/search/VitroSearchTermNames.java | 2 +- .../vitro/webapp/searchengine/solr/SolrFieldInitializer.java | 4 ++-- .../vitro/webapp/utils/searchengine/SearchQueryUtils.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/VitroSearchTermNames.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/VitroSearchTermNames.java index 73c469f86b..2822f90544 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/VitroSearchTermNames.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/VitroSearchTermNames.java @@ -72,7 +72,7 @@ public class VitroSearchTermNames { public static final String SITE_NAME = "siteName"; /** Multilingual sort field suffix */ - public static final String SORT_SUFFIX = "_label_sort"; + public static final String LABEL_SORT_SUFFIX = "_label_sort"; /** Multilingual label field suffix */ public static final String LABEL_DISPLAY_SUFFIX = "_label_display"; diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrFieldInitializer.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrFieldInitializer.java index f9cbf3c706..ce42756427 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrFieldInitializer.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrFieldInitializer.java @@ -1,7 +1,7 @@ package edu.cornell.mannlib.vitro.webapp.searchengine.solr; import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.LABEL_DISPLAY_SUFFIX; -import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.SORT_SUFFIX; +import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.LABEL_SORT_SUFFIX; import java.util.ArrayList; import java.util.Arrays; @@ -22,7 +22,7 @@ public class SolrFieldInitializer { static void initializeFields(SolrClient queryEngine, ConcurrentUpdateSolrClient updateEngine) throws Exception { - Set fieldSuffixes = new HashSet<>(Arrays.asList(SORT_SUFFIX, LABEL_DISPLAY_SUFFIX)); + Set fieldSuffixes = new HashSet<>(Arrays.asList(LABEL_SORT_SUFFIX, LABEL_DISPLAY_SUFFIX)); excludeMatchedFields(fieldSuffixes, queryEngine, "dynamicFields"); excludeMatchedFields(fieldSuffixes, queryEngine, "fields"); createMissingFields(fieldSuffixes, updateEngine); diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java index 7ede76f62b..ae99b60449 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/searchengine/SearchQueryUtils.java @@ -209,7 +209,7 @@ public static SearchQuery getQuery(List vclassUris, String alpha, } public static String getSortFieldNameForLocale(Locale locale) { - return locale.toString().replace('_', '-') + VitroSearchTermNames.SORT_SUFFIX; + return locale.toString().replace('_', '-') + VitroSearchTermNames.LABEL_SORT_SUFFIX; } public static String getLabelFieldNameForLocale(Locale locale) {