diff --git a/pom.xml b/pom.xml index 4365efbd54d..e6fd1bfb1ec 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.11.1 + 3.11.2 all @@ -189,6 +189,11 @@ turing-java-sdk ${project.version} + + com.viglet.turing + turing-commons + ${project.version} + ch.qos.logback logback-classic @@ -237,17 +242,17 @@ org.xerial sqlite-jdbc - 3.47.0.0 + 3.47.1.0 org.apache.logging.log4j log4j-core - 2.24.2 + 2.24.3 com.fasterxml.jackson.core jackson-databind - 2.18.1 + 2.18.2 org.apache.httpcomponents.client5 @@ -262,7 +267,7 @@ org.jsoup jsoup - 1.18.1 + 1.18.3 org.projectlombok diff --git a/turing-aem/aem-app/pom.xml b/turing-aem/aem-app/pom.xml index df6455e074f..892c7c0e62a 100644 --- a/turing-aem/aem-app/pom.xml +++ b/turing-aem/aem-app/pom.xml @@ -43,7 +43,7 @@ io.swagger.core.v3 swagger-annotations-jakarta - 2.2.26 + 2.2.27 compile diff --git a/turing-aem/aem-cli-sample/pom.xml b/turing-aem/aem-cli-sample/pom.xml index d451df8d68a..11cc4db1cd1 100644 --- a/turing-aem/aem-cli-sample/pom.xml +++ b/turing-aem/aem-cli-sample/pom.xml @@ -13,7 +13,7 @@ UTF-8 21 21 - 2.18.1 + 2.18.2 diff --git a/turing-aem/aem-commons/pom.xml b/turing-aem/aem-commons/pom.xml index 3d00a58719f..14fac8d116e 100644 --- a/turing-aem/aem-commons/pom.xml +++ b/turing-aem/aem-commons/pom.xml @@ -23,7 +23,7 @@ com.fasterxml.jackson.core jackson-databind - 2.18.1 + 2.18.2 com.viglet.turing diff --git a/turing-aem/aem-commons/src/main/java/com/viglet/turing/connector/aem/commons/TurAemAttrProcess.java b/turing-aem/aem-commons/src/main/java/com/viglet/turing/connector/aem/commons/TurAemAttrProcess.java index bb9c573a7f4..4ca79e9567a 100644 --- a/turing-aem/aem-commons/src/main/java/com/viglet/turing/connector/aem/commons/TurAemAttrProcess.java +++ b/turing-aem/aem-commons/src/main/java/com/viglet/turing/connector/aem/commons/TurAemAttrProcess.java @@ -44,7 +44,6 @@ public class TurAemAttrProcess { public static final String JCR_TITLE = "jcr:title"; public static final String CQ_TAGS = "cq:tags"; - public static final String TAG_SEPARATOR = ":"; public static final String DEFAULT = "default"; public static final String TEXT = "text"; public static final String JCR = "jcr:"; @@ -233,7 +232,7 @@ public TurCmsTargetAttrValueMap process(TurCmsContext context, attributeByClass(context, turAemSourceContext) : attributeByCMS(context); turCmsTargetAttrValueMap.merge(generateNewAttributesFromCqTags(context, - turAemSourceContext, turSNAttributeSpecList)); + turAemSourceContext, turSNAttributeSpecList, turCmsTargetAttrValueMap)); return turCmsTargetAttrValueMap; } @@ -252,29 +251,58 @@ private TurCmsTargetAttrValueMap attributeByCMS(TurCmsContext context) { private TurCmsTargetAttrValueMap generateNewAttributesFromCqTags(TurCmsContext context, TurAemSourceContext turAemSourceContext, - List turSNAttributeSpecList) { + List turSNAttributeSpecList, + TurCmsTargetAttrValueMap turCmsTargetAttrValueMapFromClass) { TurCmsTargetAttrValueMap turCmsTargetAttrValueMap = new TurCmsTargetAttrValueMap(); String attributeName = context.getTurCmsSourceAttr().getName(); - Object jcrProperty = getJcrProperty(context, attributeName); if (CQ_TAGS.equals(attributeName)) { - Optional.ofNullable((JSONArray) jcrProperty).ifPresent(property -> - property.forEach(tag -> { - String[] tagSplit = tag.toString().split(TAG_SEPARATOR); - if (tagSplit.length >= 2) { - Optional.ofNullable(tagSplit[0]).ifPresent(facet -> { - turSNAttributeSpecList.add(setTagFacet(turAemSourceContext, facet)); - Optional.ofNullable(tagSplit[1]).ifPresent(value -> - turCmsTargetAttrValueMap.addWithSingleValue(facet, - addTagToAttrValueList(context, turAemSourceContext, facet, value), false) - ); - }); - } - }) - ); + String targetName = context.getTurCmsTargetAttr().getName(); + if (turCmsTargetAttrValueMapFromClass.containsKey(targetName)) { + processTagsFromTargetAttr(context, turAemSourceContext, turSNAttributeSpecList, + turCmsTargetAttrValueMapFromClass, targetName, turCmsTargetAttrValueMap); + } else { + processTagsFromSourceAttr(context, turAemSourceContext, turSNAttributeSpecList, attributeName, + turCmsTargetAttrValueMap); + } } return turCmsTargetAttrValueMap; } + private static void processTagsFromSourceAttr(TurCmsContext context, TurAemSourceContext turAemSourceContext, + List turSNAttributeSpecList, + String attributeName, + TurCmsTargetAttrValueMap turCmsTargetAttrValueMap) { + Optional.ofNullable((JSONArray) getJcrProperty(context, attributeName)) + .ifPresent(property -> + property.forEach(tag -> + formatTags(context, turAemSourceContext, turSNAttributeSpecList, + tag.toString(), turCmsTargetAttrValueMap)) + ); + } + + private static void processTagsFromTargetAttr(TurCmsContext context, TurAemSourceContext turAemSourceContext, + List turSNAttributeSpecList, + TurCmsTargetAttrValueMap turCmsTargetAttrValueMapFromClass, + String targetName, + TurCmsTargetAttrValueMap turCmsTargetAttrValueMap) { + turCmsTargetAttrValueMapFromClass.get(targetName).forEach(tag -> + formatTags(context, turAemSourceContext, turSNAttributeSpecList, + tag, turCmsTargetAttrValueMap)); + } + + private static void formatTags(TurCmsContext context, TurAemSourceContext turAemSourceContext, + List turSNAttributeSpecList, String tag, + TurCmsTargetAttrValueMap turCmsTargetAttrValueMap) { + TurCommonsUtils.getKeyValueFromColon(tag).ifPresent(kv -> + Optional.ofNullable(kv.getKey()).ifPresent(facet -> { + turSNAttributeSpecList.add(setTagFacet(turAemSourceContext, facet)); + Optional.ofNullable(kv.getValue()).ifPresent(value -> + turCmsTargetAttrValueMap.addWithSingleValue(facet, + addTagToAttrValueList(context, turAemSourceContext, facet, value), false) + ); + })); + } + private TurCmsTargetAttrValueMap attributeByClass(TurCmsContext context, TurAemSourceContext turAemSourceContext) { String className = context.getTurCmsSourceAttr().getClassName(); log.debug("ClassName : {}", className); diff --git a/turing-aem/aem-commons/src/main/java/com/viglet/turing/connector/aem/commons/TurAemObject.java b/turing-aem/aem-commons/src/main/java/com/viglet/turing/connector/aem/commons/TurAemObject.java index 57dc59864dc..ef7c659a9d9 100644 --- a/turing-aem/aem-commons/src/main/java/com/viglet/turing/connector/aem/commons/TurAemObject.java +++ b/turing-aem/aem-commons/src/main/java/com/viglet/turing/connector/aem/commons/TurAemObject.java @@ -47,10 +47,11 @@ public class TurAemObject { private final String type; private final String path; private final String url; - private String model; private final JSONObject jcrNode; private JSONObject jcrContentNode = new JSONObject(); private String title; + private String template; + private String model; private final Map attributes = new HashMap<>(); public static final String CONTENT_FRAGMENT = "contentFragment"; @@ -58,6 +59,7 @@ public class TurAemObject { public static final String CQ_LAST_REPLICATION_ACTION_PUBLISH = "cq:lastReplicationAction_publish"; public static final String CQ_LAST_MODIFIED = "cq:lastModified"; public static final String CQ_MODEL = "cq:model"; + public static final String CQ_TEMPLATE = "cq:template"; public static final String DATA_FOLDER = "data"; public static final String DATE_JSON_FORMAT = "E MMM dd yyyy HH:mm:ss 'GMT'Z"; public static final String EMPTY_VALUE = ""; @@ -71,6 +73,7 @@ public TurAemObject(String nodePath, JSONObject jcrNode) { try { if (jcrNode.has(JCR_CONTENT)) { this.jcrContentNode = jcrNode.getJSONObject(JCR_CONTENT); + this.template = jcrContentNode.has(CQ_TEMPLATE) ? this.jcrContentNode.getString(CQ_TEMPLATE) : EMPTY_VALUE; this.delivered = isActivated(CQ_LAST_REPLICATION_ACTION) && isActivated(CQ_LAST_REPLICATION_ACTION_PUBLISH); this.title = jcrContentNode.has(JCR_TITLE) ? this.jcrContentNode.getString(JCR_TITLE) : EMPTY_VALUE; diff --git a/turing-app/pom.xml b/turing-app/pom.xml index 968fb9d806e..72eadfa0b4a 100644 --- a/turing-app/pom.xml +++ b/turing-app/pom.xml @@ -39,7 +39,7 @@ co.elastic.clients elasticsearch-java - 8.16.1 + 8.17.0 org.springframework.boot @@ -178,7 +178,7 @@ org.apache.opennlp opennlp-tools - 2.5.0 + 2.5.1 commons-io diff --git a/turing-app/src/main/java/com/viglet/turing/api/sn/console/TurSNSiteAPI.java b/turing-app/src/main/java/com/viglet/turing/api/sn/console/TurSNSiteAPI.java index e2abee201f0..3e1744877ad 100644 --- a/turing-app/src/main/java/com/viglet/turing/api/sn/console/TurSNSiteAPI.java +++ b/turing-app/src/main/java/com/viglet/turing/api/sn/console/TurSNSiteAPI.java @@ -139,13 +139,15 @@ public TurSNSite turSNSiteUpdate(@PathVariable String id, @RequestBody TurSNSite turSNSiteEdit.setSpotlightWithResults(turSNSite.getSpotlightWithResults()); turSNSiteEdit.setWildcardNoResults(turSNSite.getWildcardNoResults()); turSNSiteEdit.setWildcardAlways(turSNSite.getWildcardAlways()); + turSNSiteEdit.setExactMatchField(turSNSite.getExactMatchField()); + turSNSiteEdit.setDefaultField(turSNSite.getDefaultField()); turSNSiteEdit.setDefaultTitleField(turSNSite.getDefaultTitleField()); turSNSiteEdit.setDefaultTextField(turSNSite.getDefaultTextField()); turSNSiteEdit.setDefaultDescriptionField(turSNSite.getDefaultDescriptionField()); turSNSiteEdit.setDefaultDateField(turSNSite.getDefaultDateField()); turSNSiteEdit.setDefaultImageField(turSNSite.getDefaultImageField()); turSNSiteEdit.setDefaultURLField(turSNSite.getDefaultURLField()); - + turSNSiteEdit.setExactMatch(turSNSite.getExactMatch()); turSNSiteRepository.save(turSNSiteEdit); return turSNSiteEdit; }).orElse(new TurSNSite()); diff --git a/turing-app/src/main/java/com/viglet/turing/converse/TurConverse.java b/turing-app/src/main/java/com/viglet/turing/converse/TurConverse.java index e5072870744..25ea8aa6174 100644 --- a/turing-app/src/main/java/com/viglet/turing/converse/TurConverse.java +++ b/turing-app/src/main/java/com/viglet/turing/converse/TurConverse.java @@ -22,6 +22,7 @@ import com.google.inject.Inject; import com.viglet.turing.bean.converse.TurConverseAgentResponse; +import com.viglet.turing.commons.utils.TurCommonsUtils; import com.viglet.turing.persistence.model.converse.chat.TurConverseChat; import com.viglet.turing.persistence.model.converse.chat.TurConverseChatResponse; import com.viglet.turing.persistence.repository.converse.chat.TurConverseChatResponseRepository; @@ -29,7 +30,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.security.SecureRandom; @@ -71,7 +71,7 @@ public void saveChatResponseUser(String q, TurConverseChat chat, HttpSession ses String nextContext = (String) session.getAttribute(TurConverseConstants.NEXT_CONTEXT); SolrDocumentList results = turConverseSE.askPhrase(chat.getAgent(), q, nextContext); if (!results.isEmpty()) { - SolrDocument firstResult = results.get(0); + SolrDocument firstResult = results.getFirst(); String intentId = (String) firstResult.getFieldValue(TurConverseConstants.ID); SimpleEntry parameter = this.getParameterValue(q, chat, intentId); if (parameter != null) { @@ -91,11 +91,10 @@ public AbstractMap.SimpleEntry getParameterValue(String text, Tu String intentId) { SolrDocumentList termDocumentList = turConverseSE.sorlGetParameterValue(text, turConverseChat, intentId); if (!termDocumentList.isEmpty()) { - String parameter = (String) termDocumentList.get(0).getFirstValue(TurConverseConstants.PARAMETERS); - - String[] parameterKV = parameter.split(":"); - - return new AbstractMap.SimpleEntry<>(parameterKV[0], parameterKV[1]); + String parameter = (String) termDocumentList.getFirst().getFirstValue(TurConverseConstants.PARAMETERS); + return TurCommonsUtils.getKeyValueFromColon(parameter).map(p -> + new SimpleEntry<>(p.getKey(), p.getValue())) + .orElse(null); } return null; @@ -127,7 +126,6 @@ public void saveChatResponseBot(TurConverseChat chat, TurConverseAgentResponse t } - @SuppressWarnings("unchecked") public void getChatParameter(TurConverseChat chat, HttpSession session, TurConverseAgentResponse turConverseAgentResponse) { @@ -172,7 +170,7 @@ public TurConverseAgentResponse interactionNested(TurConverseChat chat, String q results = turConverseSE.askPhrase(chat.getAgent(), q, nextContext); if (!results.isEmpty()) { - SolrDocument firstResult = results.get(0); + SolrDocument firstResult = results.getFirst(); if ((boolean) firstResult.getFieldValue(TurConverseConstants.HAS_PARAMETER)) { session.setAttribute(TurConverseConstants.INTENT, firstResult.getFieldValue(TurConverseConstants.ID)); @@ -192,22 +190,17 @@ public TurConverseAgentResponse interactionNested(TurConverseChat chat, String q return turConverseAgentResponse; } - @SuppressWarnings("unchecked") private void getIntentFlow(TurConverseChat chat, HttpSession session, TurConverseAgentResponse turConverseAgentResponse, SolrDocument firstResult) { - List contextOutputs = (List) firstResult.getFieldValue(TurConverseConstants.CONTEXT_OUTPUT); - turConverseAgentResponse.setResponse(this.getIntentResponse(chat, firstResult)); turConverseAgentResponse.setIntentId(firstResult.getFieldValue(TurConverseConstants.ID).toString()); turConverseAgentResponse.setIntentName(firstResult.getFieldValue(TurConverseConstants.NAME).toString()); - if (contextOutputs != null && !contextOutputs.isEmpty()) { - session.setAttribute(TurConverseConstants.NEXT_CONTEXT, contextOutputs.get(0)); + session.setAttribute(TurConverseConstants.NEXT_CONTEXT, contextOutputs.getFirst()); } } - @SuppressWarnings("unchecked") private String getIntentResponse(TurConverseChat chat, SolrDocument firstResult) { List responses = (List) firstResult.getFieldValue(TurConverseConstants.RESPONSES); int rnd = random.nextInt(responses.size()); @@ -217,12 +210,15 @@ private String getIntentResponse(TurConverseChat chat, SolrDocument firstResult) StringBuilder responseModified = new StringBuilder(); for (String word : words) { if (word.startsWith("$")) { - String parameterName = word.replace("\\$", "").replace(",", "").replace(";", "").replace("\\.", ""); + String parameterName = word.replace("\\$", "") + .replace(",", "") + .replace(";", "") + .replace("\\.", ""); List values = turConverseChatResponseRepository .findByChatAndIsUserAndParameterNameOrderByDateDesc(chat, true, parameterName); if (!values.isEmpty()) { - word = values.get(0).getParameterValue(); + word = values.getFirst().getParameterValue(); } } responseModified.append(word).append(" "); @@ -243,7 +239,7 @@ private String getFallbackResponse(TurConverseChat chat) { if (fallbackList.isEmpty()) return TurConverseConstants.FALLBACK_DEFAULT_MESSAGE; else { - SolrDocument fallbackIntent = fallbackList.get(0); + SolrDocument fallbackIntent = fallbackList.getFirst(); Collection responses = fallbackIntent.getFieldValues(TurConverseConstants.RESPONSES); @@ -263,7 +259,7 @@ private void getIntentWhenFinishParameters(TurConverseChat chat, HttpSession ses SolrDocumentList results = turConverseSE.solrGetIntent(chat.getAgent(), intent); if (!results.isEmpty()) { - SolrDocument firstResult = results.get(0); + SolrDocument firstResult = results.getFirst(); this.getIntentFlow(chat, session, turConverseAgentResponse, firstResult); } } diff --git a/turing-app/src/main/java/com/viglet/turing/persistence/model/sn/TurSNSite.java b/turing-app/src/main/java/com/viglet/turing/persistence/model/sn/TurSNSite.java index 3fcdfcfd7e8..aa8ef3b75ff 100644 --- a/turing-app/src/main/java/com/viglet/turing/persistence/model/sn/TurSNSite.java +++ b/turing-app/src/main/java/com/viglet/turing/persistence/model/sn/TurSNSite.java @@ -81,6 +81,9 @@ public class TurSNSite extends TurAuditable implements Serializable { @Column private Integer wildcardAlways = 0; + @Column + private Integer exactMatch = 0; + @Column private Integer facet; @@ -111,6 +114,12 @@ public class TurSNSite extends TurAuditable implements Serializable { @Column private Integer thesaurus = 0; + @Column + private String defaultField; + + @Column + private String exactMatchField; + @Column private String defaultTitleField; diff --git a/turing-app/src/main/java/com/viglet/turing/sn/TurSNSearchProcess.java b/turing-app/src/main/java/com/viglet/turing/sn/TurSNSearchProcess.java index 41d05dd72d6..c8098937844 100644 --- a/turing-app/src/main/java/com/viglet/turing/sn/TurSNSearchProcess.java +++ b/turing-app/src/main/java/com/viglet/turing/sn/TurSNSearchProcess.java @@ -53,7 +53,6 @@ import com.viglet.turing.solr.TurSolr; import com.viglet.turing.solr.TurSolrInstance; import com.viglet.turing.solr.TurSolrInstanceProcess; -import com.viglet.turing.solr.TurSolrUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.SerializationUtils; import org.jetbrains.annotations.NotNull; @@ -334,41 +333,31 @@ private TurSNSiteFilterQueryBean requestFilterQuery(List fq) { private void processFilterQuery(List fq, List facetsInFilterQueries, List filterQueryModified) { if (!CollectionUtils.isEmpty(fq)) { - fq.forEach(filterQuery -> { - String[] filterParts = filterQuery.split(":"); - if (filterParts.length == 2) { - addFacetInFilterQuery(facetsInFilterQueries, filterParts); - if (!filterParts[1].startsWith("\"") && !filterParts[1].startsWith("[")) { - filterParts[1] = "\"" + filterParts[1] + "\""; - filterQueryModified.add(filterParts[0] + ":" + filterParts[1]); - } - } else { - filterQueryModified.add(filterQuery); - } - }); + fq.forEach(filterQuery -> + TurCommonsUtils.getKeyValueFromColon(filterQuery).ifPresentOrElse(f -> { + addFacetInFilterQuery(facetsInFilterQueries, f.getKey()); + if (!f.getValue().startsWith("\"") && !f.getValue().startsWith("[")) { + filterQueryModified.add("%s:\"%s\"".formatted(f.getKey(), f.getValue())); + } + }, () -> filterQueryModified.add(filterQuery))); } } - private void addFacetInFilterQuery(List facetsInFilterQueries, String[] filterParts) { - if (!facetsInFilterQueries.contains(filterParts[0])) { - facetsInFilterQueries.add(filterParts[0]); + private void addFacetInFilterQuery(List facetsInFilterQueries, String key) { + if (!facetsInFilterQueries.contains(key)) { + facetsInFilterQueries.add(key); } } public List requestTargetingRules(List tr) { List targetingRuleModified = new ArrayList<>(); if (!CollectionUtils.isEmpty(tr)) { - tr.forEach(targetingRule -> { - String[] targetingRuleParts = targetingRule.split(":"); - if (targetingRuleParts.length == 2) { - if (!targetingRuleParts[1].startsWith("\"") && !targetingRuleParts[1].startsWith("[")) { - targetingRuleParts[1] = "\"" + targetingRuleParts[1] + "\""; - targetingRuleModified.add(targetingRuleParts[0] + ":" + targetingRuleParts[1]); - } - } else { - targetingRuleModified.add(targetingRule); - } - }); + tr.forEach(targetingRule -> + TurCommonsUtils.getKeyValueFromColon(targetingRule).ifPresentOrElse(t -> { + if (!t.getValue().startsWith("\"") && !t.getValue().startsWith("[")) { + targetingRuleModified.add("%s:\"%s\"".formatted(t.getKey(), t.getValue())); + } + }, () -> targetingRuleModified.add(targetingRule))); } return targetingRuleModified; } @@ -494,7 +483,7 @@ private TurSNSiteSearchFacetBean responseFacetToRemove(TurSNSiteSearchContext co context.getTurSEParameters().getFilterQueries()); List turSNSiteSearchFacetToRemoveItemBeans = new ArrayList<>(); context.getTurSEParameters().getFilterQueries().getFq().forEach(facetToRemove -> - TurSolrUtils.getQueryKeyValue(facetToRemove).ifPresent(f -> { + TurCommonsUtils.getKeyValueFromColon(facetToRemove).ifPresent(f -> { if (turSolr.getFacetsInFilterQuery(turSNFacetTypeContext).contains(f.getKey())) { turSNSiteSearchFacetToRemoveItemBeans.add(new TurSNSiteSearchFacetItemBean() .setLabel(f.getValue().replace("\"", "")) diff --git a/turing-app/src/main/java/com/viglet/turing/sn/TurSNUtils.java b/turing-app/src/main/java/com/viglet/turing/sn/TurSNUtils.java index 51d252782d1..72df11c1696 100644 --- a/turing-app/src/main/java/com/viglet/turing/sn/TurSNUtils.java +++ b/turing-app/src/main/java/com/viglet/turing/sn/TurSNUtils.java @@ -100,9 +100,7 @@ public static URI removeFilterQuery(URI uri, String fq) { && nameValuePair.getName().equals(TurSNParamType.FILTER_QUERIES_DEFAULT))) { resetPaginationOrAddParameter(sbQueryString, nameValuePair.getName(), decodedValue); } - - - }; + } return TurCommonsUtils.modifiedURI(uri, sbQueryString); } @@ -111,7 +109,7 @@ public static URI removeFilterQueryByFieldNames(URI uri, List fieldNames StringBuilder sbQueryString = new StringBuilder(); for (NameValuePair nameValuePair : params) { if (nameValuePair.getName().equals(TurSNParamType.FILTER_QUERIES_DEFAULT)) { - TurSolrUtils.getQueryKeyValue(nameValuePair.getValue()).ifPresent(kv -> { + TurCommonsUtils.getKeyValueFromColon(nameValuePair.getValue()).ifPresent(kv -> { if (!(fieldNames.contains(java.net.URLDecoder.decode(kv.getKey(), StandardCharsets.UTF_8)))) { TurCommonsUtils.addParameterToQueryString(sbQueryString, nameValuePair.getName(), nameValuePair.getValue()); } diff --git a/turing-app/src/main/java/com/viglet/turing/sn/template/TurSNTemplate.java b/turing-app/src/main/java/com/viglet/turing/sn/template/TurSNTemplate.java index f736d0e136d..2578d26402e 100644 --- a/turing-app/src/main/java/com/viglet/turing/sn/template/TurSNTemplate.java +++ b/turing-app/src/main/java/com/viglet/turing/sn/template/TurSNTemplate.java @@ -109,6 +109,9 @@ public void defaultSNUI(TurSNSite turSNSite) { turSNSite.setSpellCheck(1); turSNSite.setSpellCheckFixes(1); turSNSite.setThesaurus(0); + turSNSite.setExactMatch(1); + turSNSite.setExactMatchField(TurSNFieldName.EXACT_MATCH); + turSNSite.setDefaultField(TurSNFieldName.DEFAULT); turSNSite.setDefaultTitleField(TurSNFieldName.TITLE); turSNSite.setDefaultTextField(TurSNFieldName.TEXT); turSNSite.setDefaultDescriptionField(TurSNFieldName.ABSTRACT); diff --git a/turing-app/src/main/java/com/viglet/turing/sn/tr/TurSNTargetingRules.java b/turing-app/src/main/java/com/viglet/turing/sn/tr/TurSNTargetingRules.java index d658842cc13..925afaf4d43 100644 --- a/turing-app/src/main/java/com/viglet/turing/sn/tr/TurSNTargetingRules.java +++ b/turing-app/src/main/java/com/viglet/turing/sn/tr/TurSNTargetingRules.java @@ -21,12 +21,12 @@ package com.viglet.turing.sn.tr; +import com.viglet.turing.commons.utils.TurCommonsUtils; import com.viglet.turing.solr.TurSolrUtils; import org.springframework.stereotype.Component; import java.util.*; import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; @Component public class TurSNTargetingRules { @@ -92,7 +92,7 @@ public String orMethod(List trs) { Set attributeList = new HashSet<>(); trs.stream().filter(tr -> tr.contains(TWO_POINTS)) .forEach(tr -> - TurSolrUtils.getQueryKeyValue(tr) + TurCommonsUtils.getKeyValueFromColon(tr) .ifPresent(kv -> { attributeList.add(String.format("NOT %s:*", kv.getKey())); trList.add(String.format("%s:%s",kv.getKey(), kv.getValue())); diff --git a/turing-app/src/main/java/com/viglet/turing/solr/TurSolr.java b/turing-app/src/main/java/com/viglet/turing/solr/TurSolr.java index 6fc51aa94a5..0aba0f00de1 100644 --- a/turing-app/src/main/java/com/viglet/turing/solr/TurSolr.java +++ b/turing-app/src/main/java/com/viglet/turing/solr/TurSolr.java @@ -62,6 +62,7 @@ import com.viglet.turing.utils.TurSNSiteFieldUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.KeyValue; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateUtils; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.ORDER; @@ -74,7 +75,6 @@ import org.apache.solr.common.params.HighlightParams; import org.apache.solr.common.params.MoreLikeThisParams; import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.tika.utils.StringUtils; import org.jetbrains.annotations.NotNull; import org.json.JSONArray; import org.springframework.stereotype.Component; @@ -372,6 +372,11 @@ private SolrQuery prepareSolrQuery(TurSNSiteSearchContext context, TurSNSite tur query.set(Q_OP, AND); setRows(turSNSite, turSEParameters); setSortEntry(turSNSite, query, turSEParameters); + + if (usesExactMatch(turSNSite, turSEParameters)) { + turSEParameters.setQuery("%s:%s".formatted(turSNSite.getExactMatchField(), turSEParameters.getQuery())); + } + if (TurSNUtils.isAutoCorrectionEnabled(context, turSNSite)) { query.setQuery(TurSNUtils.hasCorrectedText(turSESpellCheckResult) ? turSESpellCheckResult.getCorrectedText() : turSEParameters.getQuery()); @@ -391,6 +396,14 @@ private SolrQuery prepareSolrQuery(TurSNSiteSearchContext context, TurSNSite tur return query; } + private static boolean usesExactMatch(TurSNSite turSNSite, TurSEParameters turSEParameters) { + return turSEParameters.getQuery().trim().startsWith("\"") + && turSEParameters.getQuery().trim().endsWith("\"") + && !StringUtils.isEmpty(turSNSite.getExactMatchField()) + && turSNSite.getExactMatch() != null + && turSNSite.getExactMatch().equals(1); + } + private void prepareBoostQuery(TurSNSite turSNSite, SolrQuery query) { List turSNSiteFieldExtList = turSNSiteFieldExtRepository .findByTurSNSite(TurPersistenceUtils.orderByNameIgnoreCase(), turSNSite); @@ -623,7 +636,7 @@ private void setRows(TurSNSite turSNSite, TurSEParameters turSEParameters) { private void setSortEntry(TurSNSite turSNSite, SolrQuery query, TurSEParameters turSEParameters) { Optional.ofNullable(turSEParameters.getSort()).ifPresent(sort -> - TurSolrUtils.getQueryKeyValue(sort).ifPresentOrElse(kv -> + TurCommonsUtils.getKeyValueFromColon(sort).ifPresentOrElse(kv -> query.setSort(kv.getKey(), kv.getValue().equals(ASC) ? ORDER.asc : ORDER.desc), () -> { if (sort.equalsIgnoreCase(NEWEST)) @@ -921,11 +934,11 @@ private TurSNFacetMapForFilterQuery getFilterQueryMap( } private static void addEnabledFieldAsFacetItem(TurSNSiteFacetFieldEnum facetType, String fq, - List enabledFields, - TurSNFacetMapForFilterQuery facetMapForFilterQuery, TurSNSite turSNSite, - TurSEFilterQueryParameters filterQueryParameters) { + List enabledFields, + TurSNFacetMapForFilterQuery facetMapForFilterQuery, TurSNSite turSNSite, + TurSEFilterQueryParameters filterQueryParameters) { - TurSolrUtils.getQueryKeyValue(fq).flatMap(kv -> + TurCommonsUtils.getKeyValueFromColon(fq).flatMap(kv -> enabledFields.stream() .filter(facet -> facet.getName().equals(kv.getKey())) .findFirst()) @@ -1036,7 +1049,7 @@ private static TurSNFacetMapForFilterQuery getFilterQueryByDateRange( @NotNull private static List setFilterQueryRangeValue(List filterQueries, List dateFacet) { return filterQueries.stream() - .map(fq -> TurSolrUtils.getQueryKeyValue(fq) + .map(fq -> TurCommonsUtils.getKeyValueFromColon(fq) .map(facetKv -> dateFacet.stream() .filter(dateFacetItem -> facetKv.getKey().equals(dateFacetItem.getName()) && @@ -1156,7 +1169,7 @@ private static boolean isOr(TurSNFacetTypeContext context) { @NotNull private static String addDoubleQuotesToValue(String q) { - return TurSolrUtils.getQueryKeyValue(q) + return TurCommonsUtils.getKeyValueFromColon(q) .map(kv -> String.format("%s:\"%s\"", kv.getKey(), kv.getValue())) .orElse(String.format("\"%s\"", q)); } @@ -1245,7 +1258,7 @@ public List getFqFields(TurSEFilterQueryParameters queryParameters) { private static List getFqFields(List filterQueries) { return Optional.ofNullable(filterQueries) .map(fqOpt -> fqOpt.stream() - .map(fq -> TurSolrUtils.getQueryKeyValue(fq) + .map(fq -> TurCommonsUtils.getKeyValueFromColon(fq) .map(KeyValue::getKey) .orElse(null)) .toList()) diff --git a/turing-app/src/main/java/com/viglet/turing/solr/TurSolrUtils.java b/turing-app/src/main/java/com/viglet/turing/solr/TurSolrUtils.java index 1b594428467..5eec56e59bf 100644 --- a/turing-app/src/main/java/com/viglet/turing/solr/TurSolrUtils.java +++ b/turing-app/src/main/java/com/viglet/turing/solr/TurSolrUtils.java @@ -27,6 +27,7 @@ import com.jayway.jsonpath.Option; import com.viglet.turing.commons.se.TurSEParameters; import com.viglet.turing.commons.se.field.TurSEFieldType; +import com.viglet.turing.commons.utils.TurCommonsUtils; import com.viglet.turing.persistence.model.se.TurSEInstance; import com.viglet.turing.se.result.TurSEResult; import com.viglet.turing.solr.bean.TurSolrFieldBean; @@ -278,7 +279,7 @@ public static void createCollection(String solrUrl, String coreName, InputStream public static String getValueFromQuery(String q) { - return getQueryKeyValue(q).map(KeyValue::getValue).orElse(q); + return TurCommonsUtils.getKeyValueFromColon(q).map(KeyValue::getValue).orElse(q); } public static TurSEResult createTurSEResultFromDocument(SolrDocument document) { @@ -296,17 +297,6 @@ public static int lastRowPositionFromCurrentPage(TurSEParameters turSEParameters return (turSEParameters.getCurrentPage() * turSEParameters.getRows()); } - public static Optional> getQueryKeyValue(String query) { - String[] attributeKV = query.split(":"); - if (attributeKV.length >= 2) { - String key = attributeKV[0]; - String value = Arrays.stream(attributeKV).skip(1).collect(Collectors.joining(":")); - return Optional.of(new DefaultMapEntry<>(key, value)); - } else { - return Optional.empty(); - } - } - public static boolean coreExists(TurSEInstance turSEInstance, String core) { try (HttpClient client = getHttpClient()) { HttpRequest request = getHttpRequestBuilderJson() diff --git a/turing-app/src/main/resources/application.properties b/turing-app/src/main/resources/application.properties index 9ae2b028014..fdfb4a654c6 100644 --- a/turing-app/src/main/resources/application.properties +++ b/turing-app/src/main/resources/application.properties @@ -99,11 +99,10 @@ spring.servlet.multipart.max-request-size=1024MB ## Others spring.mvc.async.request-timeout=3600000 +spring.output.ansi.enabled=always jasypt.encryptor.bean=turEncryptor -spring.output.ansi.enabled=always - management.endpoints.web.exposure.include=* #spring.security.oauth2.client.registration.keycloak.client-id=demo-app diff --git a/turing-cms/cms-mapping/src/main/java/com/viglet/turing/connector/cms/beans/TurCmsContext.java b/turing-cms/cms-mapping/src/main/java/com/viglet/turing/connector/cms/beans/TurCmsContext.java index 9b2b937d75e..49761c196b5 100644 --- a/turing-cms/cms-mapping/src/main/java/com/viglet/turing/connector/cms/beans/TurCmsContext.java +++ b/turing-cms/cms-mapping/src/main/java/com/viglet/turing/connector/cms/beans/TurCmsContext.java @@ -48,4 +48,14 @@ public TurCmsContext(Object cmsObjectInstance) { this.turCmsSourceAttr = null; this.configuration = null; } + + @Override + public String toString() { + return "TurCmsContext{" + + "cmsObjectInstance=" + cmsObjectInstance + + ", turCmsTargetAttr=" + turCmsTargetAttr + + ", turCmsSourceAttr=" + turCmsSourceAttr + + ", configuration=" + configuration + + '}'; + } } diff --git a/turing-commons/pom.xml b/turing-commons/pom.xml index 80a8fcd7abe..e3ffc118f81 100644 --- a/turing-commons/pom.xml +++ b/turing-commons/pom.xml @@ -33,6 +33,11 @@ org.json json + + org.apache.commons + commons-collections4 + 4.4 + org.apache.httpcomponents.client5 httpclient5 diff --git a/turing-commons/src/main/java/com/viglet/turing/commons/file/TurFileAttributes.java b/turing-commons/src/main/java/com/viglet/turing/commons/file/TurFileAttributes.java index 009c1011d4d..4afb3ed7ff5 100644 --- a/turing-commons/src/main/java/com/viglet/turing/commons/file/TurFileAttributes.java +++ b/turing-commons/src/main/java/com/viglet/turing/commons/file/TurFileAttributes.java @@ -20,6 +20,7 @@ import lombok.*; import java.util.Date; +import java.util.HashMap; import java.util.Map; /** @@ -41,9 +42,7 @@ public class TurFileAttributes { private String name; private String title; private String extension; - private TurFileSize size; - private Date lastModified; - private Map metadata; - - + private TurFileSize size = new TurFileSize(); + private Date lastModified = new Date(); + private Map metadata = new HashMap<>(); } diff --git a/turing-commons/src/main/java/com/viglet/turing/commons/sn/field/TurSNFieldName.java b/turing-commons/src/main/java/com/viglet/turing/commons/sn/field/TurSNFieldName.java index ba1df94bcad..84d7e4569ba 100644 --- a/turing-commons/src/main/java/com/viglet/turing/commons/sn/field/TurSNFieldName.java +++ b/turing-commons/src/main/java/com/viglet/turing/commons/sn/field/TurSNFieldName.java @@ -39,4 +39,7 @@ private TurSNFieldName() { public static final String IMAGE = "image"; public static final String TYPE = "type"; public static final String SITE = "site"; + public static final String DEFAULT = "_text_"; + public static final String EXACT_MATCH = "exact_match"; + } diff --git a/turing-commons/src/main/java/com/viglet/turing/commons/utils/TurCommonsUtils.java b/turing-commons/src/main/java/com/viglet/turing/commons/utils/TurCommonsUtils.java index 17b0ad88540..3ed58f7d299 100644 --- a/turing-commons/src/main/java/com/viglet/turing/commons/utils/TurCommonsUtils.java +++ b/turing-commons/src/main/java/com/viglet/turing/commons/utils/TurCommonsUtils.java @@ -22,6 +22,8 @@ import com.viglet.turing.commons.exception.TurException; import lombok.extern.slf4j.Slf4j; import net.lingala.zip4j.ZipFile; +import org.apache.commons.collections4.KeyValue; +import org.apache.commons.collections4.keyvalue.DefaultMapEntry; import org.apache.commons.compress.archivers.ArchiveException; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.ArchiveStreamFactory; @@ -44,9 +46,8 @@ import java.nio.file.Files; import java.text.BreakIterator; import java.text.Normalizer; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; +import java.util.*; +import java.util.stream.Collectors; /** * @author Alexandre Oliveira @@ -56,11 +57,23 @@ public class TurCommonsUtils { private static final String USER_DIR = "user.dir"; private static final File userDir = new File(System.getProperty(USER_DIR)); + public static final String COLON = ":"; private TurCommonsUtils() { throw new IllegalStateException("Utility class"); } + public static Optional> getKeyValueFromColon(String stringWithColon) { + String[] attributeKV = stringWithColon.split(COLON); + if (attributeKV.length >= 2) { + String key = attributeKV[0]; + String value = Arrays.stream(attributeKV).skip(1).collect(Collectors.joining(COLON)); + return Optional.of(new DefaultMapEntry<>(key, value)); + } else { + return Optional.empty(); + } + } + public static boolean isValidUrl(URL url) { UrlValidator urlValidator = new UrlValidator(UrlValidator.ALLOW_LOCAL_URLS); diff --git a/turing-connector/.gitignore b/turing-connector/.gitignore index b83d22266ac..7fa944a21d5 100644 --- a/turing-connector/.gitignore +++ b/turing-connector/.gitignore @@ -1 +1,3 @@ /target/ +/connector-app/target/ +/connector-commons/target/ diff --git a/turing-connector/connector-app/pom.xml b/turing-connector/connector-app/pom.xml new file mode 100644 index 00000000000..c96747ea541 --- /dev/null +++ b/turing-connector/connector-app/pom.xml @@ -0,0 +1,136 @@ + + + 4.0.0 + + turing-connector + com.viglet.turing + ${revision} + + + connector-app + Turing - Connector + + + UTF-8 + UTF-8 + 21 + 21 + 21 + + + + + ch.qos.logback + logback-classic + + + com.h2database + h2 + + + org.projectlombok + lombok + true + + + com.viglet.turing + turing-spring + ${revision} + + + com.google.inject + guice + 7.0.0 + compile + + + com.fasterxml.jackson.datatype + jackson-datatype-hibernate5-jakarta + + + com.viglet.turing + turing-java-sdk + + + com.viglet.turing + wc-commons + ${revision} + + + com.sezinkarli + random-user-agent-generator + 1.3 + + + io.swagger.core.v3 + swagger-annotations-jakarta + 2.2.27 + compile + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-test + test + + + com.viglet.turing + connector-commons + 0.3.9 + compile + + + + + + org.springframework.boot + spring-boot-dependencies + ${spring-boot.version} + pom + import + + + + + turing-connector + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + true + + + + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring-boot.version} + + ZIP + false + true + + + + + repackage + + + + + + + diff --git a/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/TurConnectorApplication.java b/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/TurConnectorApplication.java new file mode 100644 index 00000000000..74fef6c0d69 --- /dev/null +++ b/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/TurConnectorApplication.java @@ -0,0 +1,39 @@ +package com.viglet.turing.connector; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.datatype.hibernate5.jakarta.Hibernate5JakartaModule; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.web.filter.CharacterEncodingFilter; + +@Slf4j +@SpringBootApplication +@EnableScheduling +public class TurConnectorApplication { + public static final String UTF_8 = "UTF-8"; + + public static void main(String[] args) { + + log.info(":: Starting Turing Connector ..."); + SpringApplication.run(TurConnectorApplication.class, args); + log.info(":: Started Turing Connector"); + } + + @Bean + FilterRegistrationBean filterRegistrationBean() { + FilterRegistrationBean registrationBean = new FilterRegistrationBean<>(); + CharacterEncodingFilter characterEncodingFilter = new CharacterEncodingFilter(); + characterEncodingFilter.setForceEncoding(true); + characterEncodingFilter.setEncoding(UTF_8); + registrationBean.setFilter(characterEncodingFilter); + return registrationBean; + } + @Bean + Module hibernate5Module() { + return new Hibernate5JakartaModule(); + } +} diff --git a/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/TurConnectorProcess.java b/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/TurConnectorProcess.java new file mode 100644 index 00000000000..31084ca2055 --- /dev/null +++ b/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/TurConnectorProcess.java @@ -0,0 +1,113 @@ +/* + * + * Copyright (C) 2016-2024 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.viglet.turing.connector; + +import com.google.common.collect.Iterators; +import com.google.inject.Inject; +import com.viglet.turing.client.auth.credentials.TurApiKeyCredentials; +import com.viglet.turing.client.sn.TurSNServer; +import com.viglet.turing.client.sn.job.TurSNJobItem; +import com.viglet.turing.client.sn.job.TurSNJobItems; +import com.viglet.turing.client.sn.job.TurSNJobUtils; +import com.viglet.turing.connector.plugin.TurConnectorPlugin; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.net.MalformedURLException; +import java.net.URI; +import java.util.*; + +@Slf4j +@Component +public class TurConnectorProcess { + public static final String ID_ATTR = "id"; + private final String turingUrl; + private final String turingApiKey; + private TurSNJobItems turSNJobItems = new TurSNJobItems(); + private final Set visitedLinks = new HashSet<>(); + private final Queue queueLinks = new LinkedList<>(); + private final int timeout; + private final int jobSize; + @Inject + public TurConnectorProcess(@Value("${turing.url}") String turingUrl, + @Value("${turing.apiKey}") String turingApiKey, + @Value("${turing.connector.timeout:5000}") int timeout, + @Value("${turing.connector.job.size:50}") int jobSize) { + this.turingUrl = turingUrl; + this.turingApiKey = turingApiKey; + this.timeout = timeout; + this.jobSize = jobSize; + } + + public void start(TurConnectorPlugin plugin) { + reset(); + TurSNJobItem currentItem = plugin.getNext(); + + if (currentItem != null) { + queueLinks.offer(currentItem); + getPagesFromQueue(plugin); + } + if (turSNJobItems.size() > 0) { + sendToTuring(); + getInfoQueue(); + } + } + private void sendToTuring() { + if (log.isDebugEnabled()) { + for (TurSNJobItem turSNJobItem : turSNJobItems) { + log.debug("TurSNJobItem Id: {}", turSNJobItem.getAttributes().get(ID_ATTR)); + } + } + try { + TurSNJobUtils.importItems(turSNJobItems, + new TurSNServer(URI.create(turingUrl).toURL(), null, + new TurApiKeyCredentials(turingApiKey)), + false); + } catch (MalformedURLException e) { + log.error(e.getMessage(), e); + } + + } + private void reset() { + turSNJobItems = new TurSNJobItems(); + visitedLinks.clear(); + } + + private void getInfoQueue() { + log.info("Total Job Item: {}", Iterators.size(turSNJobItems.iterator())); + log.info("Total Visited Links: {}", (long) visitedLinks.size()); + log.info("Queue Size: {}", (long) queueLinks.size()); + } + + public void getPagesFromQueue(TurConnectorPlugin plugin) { + while (!queueLinks.isEmpty()) { + turSNJobItems.add(queueLinks.poll()); + sendToTuringWhenMaxSize(); + getInfoQueue(); + } + } + + private void sendToTuringWhenMaxSize() { + if (turSNJobItems.size() >= jobSize) { + sendToTuring(); + turSNJobItems = new TurSNJobItems(); + } + } +} \ No newline at end of file diff --git a/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/TurConnectorScheduledTasks.java b/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/TurConnectorScheduledTasks.java new file mode 100644 index 00000000000..0096c6153ae --- /dev/null +++ b/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/TurConnectorScheduledTasks.java @@ -0,0 +1,63 @@ +/* + * + * Copyright (C) 2016-2024 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.viglet.turing.connector; + +import com.google.inject.Inject; +import com.viglet.turing.commons.cache.TurCustomClassCache; +import com.viglet.turing.connector.persistence.repository.TurConnectorConfigVarRepository; +import com.viglet.turing.connector.plugin.TurConnectorPlugin; +import com.viglet.turing.connector.webcrawler.commons.ext.TurWCExtInterface; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Component +public class TurConnectorScheduledTasks { + private final TurConnectorConfigVarRepository turConnectorConfigVarRepository; + private final TurConnectorProcess turConnectorProcess; + private final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); + public static final String FIRST_TIME = "FIRST_TIME"; + + @Inject + public TurConnectorScheduledTasks(TurConnectorConfigVarRepository turConnectorConfigVarRepository, + TurConnectorProcess turConnectorProcess) { + this.turConnectorConfigVarRepository = turConnectorConfigVarRepository; + this.turConnectorProcess = turConnectorProcess; + } + + @Scheduled(fixedDelay = 60, timeUnit = TimeUnit.MINUTES) + public void executeWebCrawler() { + TurCustomClassCache.getCustomClassMap("com.viglet.turing.connector.plugin.webcrawler.TurWCPlugin") + .ifPresent(classInstance -> { + TurConnectorPlugin turConnectorPlugin = (TurConnectorPlugin) classInstance; + log.info("The time is now {}", dateFormat.format(new Date())); + if (turConnectorConfigVarRepository.findById(FIRST_TIME).isEmpty()) { + log.info("This is the first time, waiting next schedule."); + } else { + log.info("Starting indexing"); + turConnectorProcess.start(turConnectorPlugin); + } + }); + } +} diff --git a/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/persistence/model/TurConnectorConfigVar.java b/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/persistence/model/TurConnectorConfigVar.java new file mode 100644 index 00000000000..52c6201c31c --- /dev/null +++ b/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/persistence/model/TurConnectorConfigVar.java @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2016-2022 the original author or authors. + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.viglet.turing.connector.persistence.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; + +import java.io.Serial; +import java.io.Serializable; + +/** + * The persistent class for the vigNLPSolutions database table. + * + */ +@Setter +@Getter +@Entity +@Table(name = "con_config") +public class TurConnectorConfigVar implements Serializable { + @Serial + private static final long serialVersionUID = 1L; + + @Id + @Column(unique = true, nullable = false, length = 25) + private String id; + + @Column + private String path; + + @Column + private String value; + + +} \ No newline at end of file diff --git a/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/persistence/repository/TurConnectorConfigVarRepository.java b/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/persistence/repository/TurConnectorConfigVarRepository.java new file mode 100644 index 00000000000..eb52de189b8 --- /dev/null +++ b/turing-connector/connector-app/src/main/java/com/viglet/turing/connector/persistence/repository/TurConnectorConfigVarRepository.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2016-2022 the original author or authors. + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.viglet.turing.connector.persistence.repository; + +import com.viglet.turing.connector.persistence.model.TurConnectorConfigVar; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface TurConnectorConfigVarRepository extends JpaRepository { +} diff --git a/turing-connector/connector-app/src/main/resources/application.properties b/turing-connector/connector-app/src/main/resources/application.properties new file mode 100644 index 00000000000..2598f167085 --- /dev/null +++ b/turing-connector/connector-app/src/main/resources/application.properties @@ -0,0 +1,59 @@ +# +# +# Copyright (C) 2016-2024 the original author or authors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +server.port=30130 +# Turing +turing.url=http://localhost:2700 +turing.apiKey=4618ac5e0e5640f8bd8ea8c83 +turing.connector.timeout=5000 +turing.connector.job.size=50 + +## H2 +spring.h2.console.enabled=true +spring.h2.console.path=/h2 +spring.h2.console.settings.web-allow-others=true + +## H2 - Datasource +spring.datasource.url=jdbc:h2:file:./store/db/turConnectorDB +spring.datasource.username=sa +spring.datasource.password= +spring.datasource.driver-class-name=org.h2.Driver + +## H2 - JPA +spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect +spring.jpa.properties.hibernate.globally_quoted_identifiers=true + +## JPA +spring.jpa.hibernate.ddl-auto=update +spring.jpa.show-sql=false +spring.jpa.properties.hibernate.enable_lazy_load_no_trans=true +spring.jpa.properties.jakarta.persistence.sharedCache.mode=ALL +spring.jpa.properties.hibernate.generate_statistics=false +spring.jmx.default-domain=com.viglet.turing.connector.persistence.model + +## Logging +logging.level.org.springframework=INFO +logging.level.com.viglet=INFO +logging.level.org.apache=INFO +logging.level.org.apache.activemq=ERROR +logging.file.name=store/logs/tur-connector.log +logging.logback.rollingpolicy.max-file-size=25MB +logging.logback.rollingpolicy.max-history=10 + +## Others +spring.output.ansi.enabled=always diff --git a/turing-connector/connector-commons/pom.xml b/turing-connector/connector-commons/pom.xml new file mode 100644 index 00000000000..7cb555c08fe --- /dev/null +++ b/turing-connector/connector-commons/pom.xml @@ -0,0 +1,69 @@ + + + 4.0.0 + + turing-connector + com.viglet.turing + ${revision} + + + connector-commons + Turing Connector - Commons + + + UTF-8 + 21 + 21 + + + + com.viglet.turing + turing-java-sdk + + + + + + + + maven-clean-plugin + 3.4.0 + + + + maven-resources-plugin + 3.3.1 + + + maven-compiler-plugin + 3.13.0 + + + maven-surefire-plugin + 3.5.2 + + + maven-jar-plugin + 3.4.2 + + + maven-install-plugin + 3.1.3 + + + maven-deploy-plugin + 3.1.3 + + + + maven-site-plugin + 3.21.0 + + + maven-project-info-reports-plugin + 3.8.0 + + + + + diff --git a/turing-connector/connector-commons/src/main/java/com/viglet/turing/connector/plugin/TurConnectorPlugin.java b/turing-connector/connector-commons/src/main/java/com/viglet/turing/connector/plugin/TurConnectorPlugin.java new file mode 100644 index 00000000000..60432f2e594 --- /dev/null +++ b/turing-connector/connector-commons/src/main/java/com/viglet/turing/connector/plugin/TurConnectorPlugin.java @@ -0,0 +1,26 @@ +/* + * + * Copyright (C) 2016-2024 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.viglet.turing.connector.plugin; + +import com.viglet.turing.client.sn.job.TurSNJobItem; + +public interface TurConnectorPlugin { + + TurSNJobItem getNext(); +} diff --git a/turing-connector/pom.xml b/turing-connector/pom.xml index bb1cbda70d8..154205f3515 100644 --- a/turing-connector/pom.xml +++ b/turing-connector/pom.xml @@ -1,5 +1,6 @@ - + + 4.0.0 com.viglet.turing @@ -8,38 +9,48 @@ ../pom.xml turing-connector - jar + ${revision} Turing Connector Turing Connector + pom UTF-8 UTF-8 + 21 21 21 + - org.springframework.boot - spring-boot-starter-web - - - org.apache.commons - commons-lang3 - - - org.apache.httpcomponents.client5 - httpclient5 + ch.qos.logback + logback-classic + 1.5.12 - - - - org.springframework.boot - spring-boot-dependencies - ${spring-boot.version} - import - pom - - - + + + default + + + !env.JITPACK + + + + connector-commons + connector-app + + + + jitpack + + + env.JITPACK + + + + connector-commons + + + diff --git a/turing-connector/src/main/resources/META-INF/maven/archetype.xml b/turing-connector/src/main/resources/META-INF/maven/archetype.xml deleted file mode 100644 index 766aa9cd40c..00000000000 --- a/turing-connector/src/main/resources/META-INF/maven/archetype.xml +++ /dev/null @@ -1,9 +0,0 @@ - - turing-connector - - src/main/java/App.java - - - src/test/java/AppTest.java - - diff --git a/turing-connector/src/main/resources/archetype-resources/pom.xml b/turing-connector/src/main/resources/archetype-resources/pom.xml deleted file mode 100644 index 027e773dcae..00000000000 --- a/turing-connector/src/main/resources/archetype-resources/pom.xml +++ /dev/null @@ -1,15 +0,0 @@ - - 4.0.0 - $com.viglet.turing - $turing-connector - $0.3.9 - - - junit - junit - 3.8.1 - test - - - diff --git a/turing-connector/src/main/resources/archetype-resources/src/main/java/App.java b/turing-connector/src/main/resources/archetype-resources/src/main/java/App.java deleted file mode 100644 index 63e5d53bb81..00000000000 --- a/turing-connector/src/main/resources/archetype-resources/src/main/java/App.java +++ /dev/null @@ -1,13 +0,0 @@ -package $com.viglet.turing; - -/** - * Hello world! - * - */ -public class App -{ - public static void main( String[] args ) - { - System.out.println( "Hello World!" ); - } -} diff --git a/turing-connector/src/main/resources/archetype-resources/src/test/java/AppTest.java b/turing-connector/src/main/resources/archetype-resources/src/test/java/AppTest.java deleted file mode 100644 index ceb43bfe5e1..00000000000 --- a/turing-connector/src/main/resources/archetype-resources/src/test/java/AppTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package $com.viglet.turing; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } - - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } -} diff --git a/turing-filesystem/fs-commons/pom.xml b/turing-filesystem/fs-commons/pom.xml index 94c2747c1a9..a7aebef95a1 100644 --- a/turing-filesystem/fs-commons/pom.xml +++ b/turing-filesystem/fs-commons/pom.xml @@ -82,7 +82,7 @@ org.springframework spring-web - 6.2.0 + 6.2.1 diff --git a/turing-java-sdk/pom.xml b/turing-java-sdk/pom.xml index d60c58d4194..2f3ca6de4fa 100644 --- a/turing-java-sdk/pom.xml +++ b/turing-java-sdk/pom.xml @@ -32,7 +32,7 @@ com.fasterxml.jackson.datatype jackson-datatype-jdk8 - 2.18.1 + 2.18.2 compile diff --git a/turing-react/package-lock.json b/turing-react/package-lock.json index f501299a390..533c079a01a 100644 --- a/turing-react/package-lock.json +++ b/turing-react/package-lock.json @@ -8,19 +8,19 @@ "name": "turing-react", "version": "0.3.9", "dependencies": { - "@primer/octicons-react": "^19.12.0", - "@primer/react": "^37.5.0", + "@primer/octicons-react": "^19.14.0", + "@primer/react": "^37.8.0", "@testing-library/jest-dom": "^6.6.3", - "@testing-library/react": "^16.0.1", + "@testing-library/react": "^16.1.0", "@testing-library/user-event": "^14.5.2", "@types/jest": "^29.5.14", - "@types/node": "^22.9.3", - "@types/react": "^18.3.12", - "@types/react-dom": "^18.3.1", - "axios": "^1.7.4", - "react": "^18.2.0", - "react-dom": "^18.2.0", - "react-router-dom": "^7.0.1", + "@types/node": "^22.10.2", + "@types/react": "^19.0.1", + "@types/react-dom": "^19.0.2", + "axios": "^1.7.9", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router-dom": "^7.0.2", "react-scripts": "^5.0.1", "rxjs": "^7.8.1", "styled-components": "^5.3.11", @@ -3580,9 +3580,9 @@ } }, "node_modules/@primer/octicons-react": { - "version": "19.12.0", - "resolved": "https://registry.npmjs.org/@primer/octicons-react/-/octicons-react-19.12.0.tgz", - "integrity": "sha512-xnVzFmeWqqiiv9XYZ0hrlytAlCrSZ+HM3JrRzmD/LiPgdyEhyG/qb/ewMukiDoF9Ckwy4rkd1zuDd3KAx4DIAw==", + "version": "19.14.0", + "resolved": "https://registry.npmjs.org/@primer/octicons-react/-/octicons-react-19.14.0.tgz", + "integrity": "sha512-EKeavGV7s2HYac3ybb+6vfyqHGMUeG+OlZAus5ORfEjzXlorDAIjZ59fszVJj9DI6ArfFK/Cvg8V4JRfeUHjcw==", "engines": { "node": ">=8" }, @@ -3600,9 +3600,9 @@ } }, "node_modules/@primer/react": { - "version": "37.5.0", - "resolved": "https://registry.npmjs.org/@primer/react/-/react-37.5.0.tgz", - "integrity": "sha512-HRCAdHNJvuL59ejD5vDPfsPnckvArqDYcEBk73Db3GwBnJHCCWlci0sUPKLJD6QWWd8zVMEefaJLV2/cmnet2Q==", + "version": "37.8.0", + "resolved": "https://registry.npmjs.org/@primer/react/-/react-37.8.0.tgz", + "integrity": "sha512-q8Q5gvmWGHuXkv2p4lv7WhRlraOmJZqB8cWAUmMQl4wmLHTZMwKe5kj6dHkKmoZ7pEsNYM5PdkJ1Ize45cLLRA==", "dependencies": { "@github/relative-time-element": "^4.4.3", "@github/tab-container-element": "^4.8.0", @@ -4152,9 +4152,9 @@ "integrity": "sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w==" }, "node_modules/@testing-library/react": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-16.0.1.tgz", - "integrity": "sha512-dSmwJVtJXmku+iocRhWOUFbrERC76TX2Mnf0ATODz8brzAZrMBbzLwQixlBSanZxR6LddK3eiwpSFZgDET1URg==", + "version": "16.1.0", + "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-16.1.0.tgz", + "integrity": "sha512-Q2ToPvg0KsVL0ohND9A3zLJWcOXXcO8IDu3fj11KhNt0UlCWyFyvnCIBkd12tidB2lkiVRG8VFqdhcqhqnAQtg==", "dependencies": { "@babel/runtime": "^7.12.5" }, @@ -4163,10 +4163,10 @@ }, "peerDependencies": { "@testing-library/dom": "^10.0.0", - "@types/react": "^18.0.0", - "@types/react-dom": "^18.0.0", - "react": "^18.0.0", - "react-dom": "^18.0.0" + "@types/react": "^18.0.0 || ^19.0.0", + "@types/react-dom": "^18.0.0 || ^19.0.0", + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" }, "peerDependenciesMeta": { "@types/react": { @@ -4430,11 +4430,11 @@ "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==" }, "node_modules/@types/node": { - "version": "22.9.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.9.3.tgz", - "integrity": "sha512-F3u1fs/fce3FFk+DAxbxc78DF8x0cY09RRL8GnXLmkJ1jvx3TtPdWoTT5/NiYfI5ASqXBmfqJi9dZ3gxMx4lzw==", + "version": "22.10.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", + "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", "dependencies": { - "undici-types": "~6.19.8" + "undici-types": "~6.20.0" } }, "node_modules/@types/node-forge": { @@ -4455,11 +4455,6 @@ "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.7.3.tgz", "integrity": "sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA==" }, - "node_modules/@types/prop-types": { - "version": "15.7.13", - "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.13.tgz", - "integrity": "sha512-hCZTSvwbzWGvhqxp/RqVqwU999pBf2vp7hzIjiYOsl8wqOmUxkQ6ddw1cV3l8811+kdUFus/q4d1Y3E3SyEifA==" - }, "node_modules/@types/q": { "version": "1.5.8", "resolved": "https://registry.npmjs.org/@types/q/-/q-1.5.8.tgz", @@ -4476,20 +4471,19 @@ "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==" }, "node_modules/@types/react": { - "version": "18.3.12", - "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.12.tgz", - "integrity": "sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw==", + "version": "19.0.1", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.0.1.tgz", + "integrity": "sha512-YW6614BDhqbpR5KtUYzTA+zlA7nayzJRA9ljz9CQoxthR0sDisYZLuvSMsil36t4EH/uAt8T52Xb4sVw17G+SQ==", "dependencies": { - "@types/prop-types": "*", "csstype": "^3.0.2" } }, "node_modules/@types/react-dom": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.1.tgz", - "integrity": "sha512-qW1Mfv8taImTthu4KoXgDfLuk4bydU6Q/TkADnDWWHwi4NX4BR+LWfTp2sVmTqRrsHvyDDTelgelxJ+SsejKKQ==", - "dependencies": { - "@types/react": "*" + "version": "19.0.2", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.0.2.tgz", + "integrity": "sha512-c1s+7TKFaDRRxr1TxccIX2u7sfCnc3RxkVyBIUA2lCpyqCF+QoAwQ/CBg7bsMdVwP120HEH143VQezKtef5nCg==", + "peerDependencies": { + "@types/react": "^19.0.0" } }, "node_modules/@types/react-is": { @@ -5411,9 +5405,9 @@ } }, "node_modules/axios": { - "version": "1.7.7", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", - "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "version": "1.7.9", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.9.tgz", + "integrity": "sha512-LhLcE7Hbiryz8oMDdDptSrWowmB4Bl6RCt6sIJKpRB4XtVf0iEgewX3au/pJqm+Py1kCASkb/FFKjxQaLtxJvw==", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -14457,12 +14451,9 @@ } }, "node_modules/react": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", - "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", - "dependencies": { - "loose-envify": "^1.1.0" - }, + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.0.0.tgz", + "integrity": "sha512-V8AVnmPIICiWpGfm6GLzCR/W5FXLchHop40W4nXBmdlEceh16rCN8O8LNWm5bh5XUX91fh7KpA+W0TgMKmgTpQ==", "engines": { "node": ">=0.10.0" } @@ -14531,15 +14522,14 @@ } }, "node_modules/react-dom": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", - "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.0.0.tgz", + "integrity": "sha512-4GV5sHFG0e/0AD4X+ySy6UJd3jVl1iNsNHdpad0qhABJ11twS3TTBnseqsKurKcsNqCEFeGL3uLpVChpIO3QfQ==", "dependencies": { - "loose-envify": "^1.1.0", - "scheduler": "^0.23.2" + "scheduler": "^0.25.0" }, "peerDependencies": { - "react": "^18.3.1" + "react": "^19.0.0" } }, "node_modules/react-error-overlay": { @@ -14575,9 +14565,9 @@ } }, "node_modules/react-router": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.0.1.tgz", - "integrity": "sha512-WVAhv9oWCNsja5AkK6KLpXJDSJCQizOIyOd4vvB/+eHGbYx5vkhcmcmwWjQ9yqkRClogi+xjEg9fNEOd5EX/tw==", + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.0.2.tgz", + "integrity": "sha512-m5AcPfTRUcjwmhBzOJGEl6Y7+Crqyju0+TgTQxoS4SO+BkWbhOrcfZNq6wSWdl2BBbJbsAoBUb8ZacOFT+/JlA==", "dependencies": { "@types/cookie": "^0.6.0", "cookie": "^1.0.1", @@ -14598,11 +14588,11 @@ } }, "node_modules/react-router-dom": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.0.1.tgz", - "integrity": "sha512-duBzwAAiIabhFPZfDjcYpJ+f08TMbPMETgq254GWne2NW1ZwRHhZLj7tpSp8KGb7JvZzlLcjGUnqLxpZQVEPng==", + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.0.2.tgz", + "integrity": "sha512-VJOQ+CDWFDGaWdrG12Nl+d7yHtLaurNgAQZVgaIy7/Xd+DojgmYLosFfZdGz1wpxmjJIAkAMVTKWcvkx1oggAw==", "dependencies": { - "react-router": "7.0.1" + "react-router": "7.0.2" }, "engines": { "node": ">=20.0.0" @@ -15557,12 +15547,9 @@ } }, "node_modules/scheduler": { - "version": "0.23.2", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", - "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", - "dependencies": { - "loose-envify": "^1.1.0" - } + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.25.0.tgz", + "integrity": "sha512-xFVuu11jh+xcO7JOAGJNOXld8/TcEHK/4CituBUeUb5hqxJLj9YuemAEuvm9gQ/+pgXYfbQuqAkiYu+u7YEsNA==" }, "node_modules/schema-utils": { "version": "4.2.0", @@ -17232,9 +17219,10 @@ "integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==" }, "node_modules/undici-types": { - "version": "6.19.8", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", - "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==" + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "license": "MIT" }, "node_modules/unicode-canonical-property-names-ecmascript": { "version": "2.0.1", diff --git a/turing-react/package.json b/turing-react/package.json index f92c393a196..4b17fcedca9 100644 --- a/turing-react/package.json +++ b/turing-react/package.json @@ -3,23 +3,23 @@ "version": "0.3.9", "private": true, "dependencies": { - "@primer/octicons-react": "^19.12.0", - "@primer/react": "^37.5.0", + "@primer/octicons-react": "^19.14.0", + "@primer/react": "^37.8.0", "@testing-library/jest-dom": "^6.6.3", - "@testing-library/react": "^16.0.1", + "@testing-library/react": "^16.1.0", "@testing-library/user-event": "^14.5.2", "@types/jest": "^29.5.14", - "@types/node": "^22.9.3", - "@types/react": "^18.3.12", - "@types/react-dom": "^18.3.1", - "axios": "^1.7.4", - "react": "^18.2.0", - "react-dom": "^18.2.0", + "@types/node": "^22.10.2", + "@types/react": "^19.0.1", + "@types/react-dom": "^19.0.2", + "axios": "^1.7.9", + "react": "^19.0.0", + "react-dom": "^19.0.0", "react-scripts": "^5.0.1", "rxjs": "^7.8.1", "styled-components": "^5.3.11", "typescript": "^4.1.6", - "react-router-dom": "^7.0.1" + "react-router-dom": "^7.0.2" }, "scripts": { "start": "react-scripts start", diff --git a/turing-sprinklr/pom.xml b/turing-sprinklr/pom.xml index da2df3cfef6..c07c99df25b 100644 --- a/turing-sprinklr/pom.xml +++ b/turing-sprinklr/pom.xml @@ -12,9 +12,6 @@ ${revision} Turing Sprinklr Turing Sprinklr Connector - - sprinklr-java-sdk - pom UTF-8 diff --git a/turing-sprinklr/sprinklr-app/pom.xml b/turing-sprinklr/sprinklr-app/pom.xml index e170b60cde6..1c0b4144bde 100644 --- a/turing-sprinklr/sprinklr-app/pom.xml +++ b/turing-sprinklr/sprinklr-app/pom.xml @@ -44,7 +44,7 @@ io.swagger.core.v3 swagger-annotations-jakarta - 2.2.26 + 2.2.27 compile diff --git a/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/TurSprinklrApplication.java b/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/TurSprinklrApplication.java index ec196870e1e..4b5f185c470 100644 --- a/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/TurSprinklrApplication.java +++ b/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/TurSprinklrApplication.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.datatype.hibernate5.jakarta.Hibernate5JakartaModule; +import com.viglet.turing.connector.sprinklr.commons.plugins.TurSprinklrKeyValueTransformer; +import com.viglet.turing.connector.sprinklr.commons.plugins.TurSprinklrPluginContext; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.FilterRegistrationBean; @@ -31,4 +33,16 @@ FilterRegistrationBean filterRegistrationBean() { Module hibernate5Module() { return new Hibernate5JakartaModule(); } + + @Bean + public TurSprinklrPluginContext pluginContext(TurSprinklrKeyValueTransformer transformer) { + TurSprinklrPluginContext pluginContext = new TurSprinklrPluginContext(); + pluginContext.addPlugin(transformer); + return pluginContext; + } + + @Bean + public TurSprinklrKeyValueTransformer keyValueTransformer() { + return new TurSprinklrKeyValueTransformer(); + } } diff --git a/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/TurSprinklrProcess.java b/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/TurSprinklrProcess.java index 66c26b64fd2..1d0999af3b9 100644 --- a/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/TurSprinklrProcess.java +++ b/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/TurSprinklrProcess.java @@ -17,6 +17,7 @@ import com.viglet.turing.connector.sprinklr.persistence.model.TurSprinklrAttributeMapping; import com.viglet.turing.connector.sprinklr.persistence.model.TurSprinklrSource; import com.viglet.turing.connector.sprinklr.persistence.repository.TurSprinklrAttributeMappingRepository; +import com.viglet.turing.connector.sprinklr.commons.plugins.TurSprinklrPluginContext; import com.viglet.turing.connector.sprinklr.utils.FileAsset; import com.viglet.turing.connector.sprinklr.utils.FileAssetsExtractor; import com.viglet.turing.sprinklr.client.service.kb.TurSprinklrKBService; @@ -47,15 +48,20 @@ public class TurSprinklrProcess { */ private TurSNJobItems turSNJobItems = new TurSNJobItems(); + private final TurSprinklrPluginContext pluginContext; + @Inject public TurSprinklrProcess(@Value("${turing.url}") String turingUrl, @Value("${turing.apiKey}") String turingApiKey, @Value("${turing.sprinklr.job.size}") int jobSize, - TurSprinklrAttributeMappingRepository turSprinklrAttributeMappingRepository) { + TurSprinklrAttributeMappingRepository turSprinklrAttributeMappingRepository, + TurSprinklrPluginContext pluginContext + ) { this.turingUrl = turingUrl; this.turingApiKey = turingApiKey; this.jobSize = jobSize; this.turSprinklrAttributeMappingRepository = turSprinklrAttributeMappingRepository; + this.pluginContext = pluginContext; } private static void addItemInExistingAttribute(String attributeValue, @@ -101,7 +107,6 @@ public void start(TurSprinklrSource turSprinklrSource) { final var fileAssetExtractor = new FileAssetsExtractor(turingUrl, turingApiKey); while (true) { TurSprinklrKBSearch turSprinklrKBSearch = TurSprinklrKBService.run(turSprinklrAccessToken, kbPage.get()); - if (turSprinklrKBSearch != null) { List results = turSprinklrKBSearch.getData().getSearchResults(); if (results.isEmpty()) { @@ -116,9 +121,9 @@ public void start(TurSprinklrSource turSprinklrSource) { // Gets the assets attached to the search result and inserts into turSNJobItems. List assets = getFileAssets(searchResult, fileAssetExtractor); - addFileAssetsToJobItens(assets, resultLocale, turSites); + addFileAssetsToJobItems(assets, resultLocale, turSites); - // Quando o tamanho de turSNJobItems alcançar o JobSize definido, envia para o turing. + // When the size of turSNJobItems reaches the defined JobSize, send it to turing. sendToTuringWhenMaxSize(); getInfoQueue(); @@ -151,7 +156,7 @@ private List getFileAssets(TurSprinklrSearchResult searchResult, File /** * Adds the file assets to the job items list. */ - private void addFileAssetsToJobItens(List fileAssets, Locale locale, Collection turSites) { + private void addFileAssetsToJobItems(List fileAssets, Locale locale, Collection turSites) { for (var asset : fileAssets) { var turSNJobItemAttributes = asset.toMapAttributes(); TurSNJobItem turSNJobItem = new TurSNJobItem( @@ -288,6 +293,7 @@ private void setAttribute(TurSprinklrAttributeMapping attribute, String attribut private Optional getCustomClass(TurSprinklrSearchResult searchResult, TurSprinklrAccessToken token, TurSprinklrAttributeMapping turSprinklrAttributeMapping) { + log.debug(getTurSprinklrContext(searchResult, token).getPluginContext().toString()); return TurCustomClassCache.getCustomClassMap(turSprinklrAttributeMapping.getClassName()) .flatMap(classInstance -> ((TurSprinklrExtInterface) classInstance) .consume(getTurSprinklrContext(searchResult, token))); @@ -300,6 +306,7 @@ public TurSprinklrContext getTurSprinklrContext(TurSprinklrSearchResult searchRe return TurSprinklrContext.builder() .searchResult(searchResult) .accessToken(token) + .pluginContext(pluginContext) .build(); } @@ -324,7 +331,7 @@ private void sendToTuring() { TurSNJobUtils.importItems(turSNJobItems, new TurSNServer(URI.create(turingUrl).toURL(), null, new TurApiKeyCredentials(turingApiKey)), - false); + true); } catch (MalformedURLException e) { log.error(e.getMessage(), e); } diff --git a/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/export/TurSprinklrExchangeProcess.java b/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/export/TurSprinklrExchangeProcess.java index 0725065a1d2..63784abbff7 100644 --- a/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/export/TurSprinklrExchangeProcess.java +++ b/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/export/TurSprinklrExchangeProcess.java @@ -86,6 +86,7 @@ public StreamingResponseBody exportObject(HttpServletResponse response) { .locale(turSprinklrSource.getLocale()) .localeClass(turSprinklrSource.getLocaleClass()) .turSNSites(turSprinklrSource.getTurSNSites()) + .tagMapping(turSprinklrSource.getTagMapping()) .build()).toList())); File zipFile = new File(tmpDir.getAbsolutePath().concat(File.separator + folderName + ".zip")); @@ -161,7 +162,7 @@ public void importSprinklrSource(TurSprinklrExchange turSprinklrExchange) { for (TurSprinklrSourceExchange turSprinklrSourceExchange : turSprinklrExchange.getSources()) { if (turSprinklrSourceRepository.findById(turSprinklrSourceExchange.getId()).isEmpty()) { TurSprinklrSource turSprinklrSource = TurSprinklrSource.builder() - .id(turSprinklrSourceExchange.getId()) +// .id(turSprinklrSourceExchange.getId()) evitar exception .url(turSprinklrSourceExchange.getUrl()) .turSNSites(turSprinklrSourceExchange.getTurSNSites()) .locale(turSprinklrSourceExchange.getLocale()) diff --git a/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/export/bean/TurSprinklrSourceExchange.java b/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/export/bean/TurSprinklrSourceExchange.java index 86fdcbba43c..4d95f10cfec 100644 --- a/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/export/bean/TurSprinklrSourceExchange.java +++ b/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/export/bean/TurSprinklrSourceExchange.java @@ -11,6 +11,7 @@ @NoArgsConstructor @Getter @Setter +@ToString public class TurSprinklrSourceExchange { private String id; private Locale locale; @@ -19,6 +20,7 @@ public class TurSprinklrSourceExchange { private String environment; private String secretKey; private String apiKey; + private String tagMapping; @Builder.Default private Collection turSNSites = new HashSet<>(); @Builder.Default diff --git a/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/persistence/model/TurSprinklrSource.java b/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/persistence/model/TurSprinklrSource.java index fda306a5f30..402007c883e 100644 --- a/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/persistence/model/TurSprinklrSource.java +++ b/turing-sprinklr/sprinklr-app/src/main/java/com/viglet/turing/connector/sprinklr/persistence/model/TurSprinklrSource.java @@ -48,7 +48,8 @@ public class TurSprinklrSource implements Serializable { private String apiKey; @Column private String secretKey; - + @Column + private String tagMapping; @Builder.Default @ElementCollection(targetClass = String.class, fetch = FetchType.EAGER) @CollectionTable(name = "sprinklr_sn_site", joinColumns = @JoinColumn(name = "source_id")) diff --git a/turing-sprinklr/sprinklr-app/src/test/java/com/viglet/turing/connector/sprinklr/plugins/TurSprinklrKeyValueTransformerTest.java b/turing-sprinklr/sprinklr-app/src/test/java/com/viglet/turing/connector/sprinklr/plugins/TurSprinklrKeyValueTransformerTest.java new file mode 100644 index 00000000000..0d662ff11c6 --- /dev/null +++ b/turing-sprinklr/sprinklr-app/src/test/java/com/viglet/turing/connector/sprinklr/plugins/TurSprinklrKeyValueTransformerTest.java @@ -0,0 +1,95 @@ +package com.viglet.turing.connector.sprinklr.plugins; + +import com.viglet.turing.connector.sprinklr.commons.plugins.TurSprinklrKeyValueTransformer; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +@SpringBootTest +class TurSprinklrKeyValueTransformerTest { + + @Autowired + private TurSprinklrKeyValueTransformer transformer; + + @Test + void loadMapping() { + // Path to test file (adjust to correct location) + String testFile = "src/test/resources/test-mapping.txt"; + + // Call the loadMapping method + transformer.loadMapping(testFile); + + // Checks if the mapping has been loaded + Map mapping = transformer.getMapping(testFile); + assertNotNull(mapping, "Mapping should not be null after loading"); + assertFalse(mapping.isEmpty(), "Mapping should not be empty after loading"); + + // Checks if a specific key was loaded correctly + assertEquals("action", mapping.get("424123421312"), "Value for key '424123421312' should be 'action'"); + } + + @Test + void getMapping() { + // Path to test file (adjust to correct location) + String testFile = "src/test/resources/test-mapping.txt"; + + // Make sure the mapping has been loaded + transformer.loadMapping(testFile); + + // Retrieves the mapping and checks + Map mapping = transformer.getMapping(testFile); + assertNotNull(mapping, "Mapping should not be null"); + // There are two lines with the same key + assertEquals(2, mapping.size(), "Mapping should have 2 entries"); + + // Checks if a specific key is present + assertTrue(mapping.containsKey("235521957944"), "Mapping should contain key '235521957944'"); + } + + @Test + void transform() { + // Path to test file (adjust to correct location) + String testFile = "src/test/resources/test-mapping.txt"; + + // Make sure the mapping has been loaded + transformer.loadMapping(testFile); + + // Test the transformation + String transformedValue = transformer.transform("424123421312", testFile); + assertEquals("action", transformedValue, "Transformed value should be 'action'"); + } + + @Test + void loadMappingTransFromLoadSameMapTransform() { + // Path to test file (adjust to correct location) + String testFile = "src/test/resources/test-mapping.txt"; + + // Call the loadMapping method + transformer.loadMapping(testFile); + + var transformedValue = transformer.transform("424123421312", testFile); + assertEquals("action", transformedValue, "Transformed value should be 'action'"); + transformer.loadMapping(testFile); + transformedValue = transformer.transform("235521957944", testFile); + assertEquals("adventure", transformedValue, "Transformed value should be 'adventure'"); + } + + @Test + void getName() { + String name = transformer.getName(); + assertNotNull(name); + assertEquals("Key Value Transformer Plugin", transformer.getName()); + + } + + @Test + void getDescription() { + String description = transformer.getDescription(); + assertNotNull(description); + assertEquals("Transforms a value into another value based on a key-value mapping file", transformer.getDescription()); + } +} \ No newline at end of file diff --git a/turing-sprinklr/sprinklr-app/src/test/java/com/viglet/turing/connector/sprinklr/plugins/TurSprinklrPluginContextTest.java b/turing-sprinklr/sprinklr-app/src/test/java/com/viglet/turing/connector/sprinklr/plugins/TurSprinklrPluginContextTest.java new file mode 100644 index 00000000000..a2fb5bc7de2 --- /dev/null +++ b/turing-sprinklr/sprinklr-app/src/test/java/com/viglet/turing/connector/sprinklr/plugins/TurSprinklrPluginContextTest.java @@ -0,0 +1,20 @@ +package com.viglet.turing.connector.sprinklr.plugins; + +import com.viglet.turing.connector.sprinklr.commons.plugins.TurSprinklrKeyValueTransformer; +import com.viglet.turing.connector.sprinklr.commons.plugins.TurSprinklrPluginContext; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest () +class TurSprinklrPluginContextTest { + + @Autowired + private TurSprinklrPluginContext pluginContext; + + @Test + void testGetPlugin() { + pluginContext.getPlugin(TurSprinklrKeyValueTransformer.class); + } + +} \ No newline at end of file diff --git a/turing-sprinklr/sprinklr-app/src/test/resources/test-mapping.txt b/turing-sprinklr/sprinklr-app/src/test/resources/test-mapping.txt new file mode 100644 index 00000000000..afbe65a1c42 --- /dev/null +++ b/turing-sprinklr/sprinklr-app/src/test/resources/test-mapping.txt @@ -0,0 +1,3 @@ +424123421312 action +235521957944 adventure +235521957944 comedy \ No newline at end of file diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/TurSprinklrContext.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/TurSprinklrContext.java deleted file mode 100644 index bc10d7c7c9c..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/TurSprinklrContext.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.viglet.turing.connector.sprinklr; - -import com.viglet.turing.sprinklr.client.service.kb.response.TurSprinklrSearchResult; -import com.viglet.turing.sprinklr.client.service.token.TurSprinklrAccessToken; -import lombok.Builder; -import lombok.Getter; -import lombok.Setter; - -@Builder -@Getter -@Setter -public class TurSprinklrContext { - private TurSprinklrSearchResult searchResult; - private TurSprinklrAccessToken accessToken; - -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/TurSprinklrContext.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/TurSprinklrContext.java index e5225a25bb3..3a109b98276 100644 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/TurSprinklrContext.java +++ b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/TurSprinklrContext.java @@ -2,15 +2,18 @@ import com.viglet.turing.sprinklr.client.service.kb.response.TurSprinklrSearchResult; import com.viglet.turing.sprinklr.client.service.token.TurSprinklrAccessToken; +import com.viglet.turing.connector.sprinklr.commons.plugins.TurSprinklrPluginContext; import lombok.Builder; import lombok.Getter; import lombok.Setter; +import lombok.ToString; @Builder @Getter @Setter +@ToString public class TurSprinklrContext { private TurSprinklrSearchResult searchResult; private TurSprinklrAccessToken accessToken; - + private TurSprinklrPluginContext pluginContext; } diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/plugins/TurSprinklrKeyValueTransformer.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/plugins/TurSprinklrKeyValueTransformer.java new file mode 100644 index 00000000000..1851eaffc90 --- /dev/null +++ b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/plugins/TurSprinklrKeyValueTransformer.java @@ -0,0 +1,147 @@ +/* + * + * Copyright (C) 2016-2024 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.viglet.turing.connector.sprinklr.commons.plugins; + +import com.viglet.turing.commons.exception.TurRuntimeException; +import lombok.extern.slf4j.Slf4j; + +import java.io.*; +import java.util.*; + +/** + * @author Gabriel F. Gomazako + * @since 0.3.9 + */ +@Slf4j +public class TurSprinklrKeyValueTransformer implements TurSprinklrPlugin { + + private final Set files = new HashSet<>(); + + private final Map> mappings = new HashMap<>(); + + + + TurSprinklrNotFound notFoundAction = TurSprinklrNotFound.DEFAULT_VALUE; + + String defaultNotFoundText = "Key Value Transformer Plugin: Key not found"; + + public void loadMapping(String file) { + + if (!files.contains(file)) { + files.add(file); + var newMapping = createMappingFromFile(file); + mappings.putIfAbsent(file, newMapping); + } + } + + /** + * Creates a mapping from a file. Example of the structure of the file: + * + * 
+     * 424123421312 action
+     * 235521957944 adventure
+     * 235521957944 comedy
+     *
+ * + * @param file The file to create the mapping from + * @return a map containing the key-value pairs from the file + */ + private Map createMappingFromFile(String file) { + File mappingFile = new File(file); + HashMap newMapping = new HashMap<>(); + try { + BufferedReader reader = new BufferedReader(new FileReader(mappingFile)); + + while (reader.ready()) { + String line = reader.readLine(); + line = line.trim(); + String[] keyValue = line.split(" "); + + if (keyValue.length != 2) { + reader.close(); + throw new TurRuntimeException("Invalid line: " + line); + } + + if (newMapping.putIfAbsent(keyValue[0], keyValue[1]) != null) { + log.warn("Duplicated key: {}", keyValue[0]); + } + } + reader.close(); + return newMapping; + } catch (FileNotFoundException e) { + log.error("Mapping file not found: {}", file, e); + throw new IllegalArgumentException("The specified file does not exist: " + file, e); + } catch (IOException e) { + log.error("Error reading file: {}", file, e); + throw new UncheckedIOException("Error occurred while reading the file: " + file, e); + } + } + + public Map getMapping(String file) { + return mappings.get(file); + } + + /** + * Transforms the given key using the mapping obtained from the specified file. + * + * @param key the key to be transformed + * @param file the file from which the mapping is obtained + * @return the transformed value corresponding to the key, or a treated value if the key is not found + */ + public String transform(String key, String file) { + Map mapping = getMapping(file); + return mapping.getOrDefault(key, getKey(key, file)); + } + + private String getKey(String key, String file) { + + if (Objects.requireNonNull(notFoundAction) == TurSprinklrNotFound.KEEP_KEY) { + log.info("Key found: {} in file: {}", key, file); + return key; + } else { + log.info("Key not found: {} in file: {}", key, file); + } + return null; + } + + public void configure(TurSprinklrNotFound action) { + notFoundAction = action; + } + + /** + * Configures the action to be taken when a key is not found and sets the default text for such cases. + * + * @param action the action to be taken when a key is not found + * @param defaultText the default text to be used when a key is not found + */ + public void configure(TurSprinklrNotFound action, String defaultText) { + notFoundAction = action; + defaultNotFoundText = defaultText; + } + + @Override + public String getName() { + return "Key Value Transformer Plugin"; + } + + @Override + public String getDescription() { + return "Transforms a value into another value based on a key-value mapping file"; + } +} \ No newline at end of file diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/plugins/TurSprinklrNotFound.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/plugins/TurSprinklrNotFound.java new file mode 100644 index 00000000000..d7ed3b4dba3 --- /dev/null +++ b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/plugins/TurSprinklrNotFound.java @@ -0,0 +1,32 @@ +/* + * + * Copyright (C) 2016-2024 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.viglet.turing.connector.sprinklr.commons.plugins; +/** + * Enum representing the actions to take when a key is not found. + *
    + *
  • DEFAULT_VALUE: Returns a default value
    • + *
  • KEEP_KEY: Returns the key itself
    • + *
  • NULL: Returns null
    • + *
+ */ +public enum TurSprinklrNotFound { + DEFAULT_VALUE, + KEEP_KEY, + NULL +} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/plugins/TurSprinklrPlugin.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/plugins/TurSprinklrPlugin.java new file mode 100644 index 00000000000..a66b02d69f0 --- /dev/null +++ b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/plugins/TurSprinklrPlugin.java @@ -0,0 +1,36 @@ +/* + * + * Copyright (C) 2016-2024 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.viglet.turing.connector.sprinklr.commons.plugins; + + +public interface TurSprinklrPlugin { + /** + * Retrieves the name associated with this plugin. + * + * @return the name of the plugin + */ + String getName(); + /** + * Retrieves the description of the plugin. + * + * @return a String representing the description of the plugin. + */ + String getDescription(); + +} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/plugins/TurSprinklrPluginContext.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/plugins/TurSprinklrPluginContext.java new file mode 100644 index 00000000000..7ceaa9814ca --- /dev/null +++ b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/plugins/TurSprinklrPluginContext.java @@ -0,0 +1,50 @@ +/* + * + * Copyright (C) 2016-2024 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.viglet.turing.connector.sprinklr.commons.plugins; + +import com.viglet.turing.commons.exception.TurRuntimeException; +import lombok.ToString; + +import java.util.ArrayList; +import java.util.List; + +@ToString +public class TurSprinklrPluginContext { + List initializedPlugins = new ArrayList<>(); + + public TurSprinklrPluginContext(List initializedPlugins){ + this.initializedPlugins = initializedPlugins; + } + + public TurSprinklrPluginContext(){ + } + + public void addPlugin(TurSprinklrPlugin plugin){ + initializedPlugins.add(plugin); + } + + public T getPlugin(Class pluginClass){ + for (TurSprinklrPlugin plugin : initializedPlugins){ + if (pluginClass.isInstance(plugin)){ + return pluginClass.cast(plugin); + } + } + throw new TurRuntimeException("Plugin not found" + pluginClass.getName()); + } +} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/tools/KeyValueTransformerTool.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/tools/KeyValueTransformerTool.java new file mode 100644 index 00000000000..2c8adf91bc7 --- /dev/null +++ b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/commons/tools/KeyValueTransformerTool.java @@ -0,0 +1,33 @@ +package com.viglet.turing.connector.sprinklr.commons.tools; + +import com.viglet.turing.connector.sprinklr.commons.plugins.TurSprinklrKeyValueTransformer; +import com.viglet.turing.connector.sprinklr.commons.plugins.TurSprinklrPluginContext; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class KeyValueTransformerTool { + private final TurSprinklrPluginContext pluginContext; + + public KeyValueTransformerTool(TurSprinklrPluginContext pluginContext) { + this.pluginContext = pluginContext; + } + /** + * Transforms a key into a value using the `KeyValueTransformer` and the specified file. + * + * @param key The key that will be transformed. + * @param file The name of the file containing the mapping. + * @return The transformed value, or null if the value is not found. + */ + public String transform(String key, String file) { + // Retrieves the KeyValueTransformer plugin + TurSprinklrKeyValueTransformer transformer = pluginContext.getPlugin(TurSprinklrKeyValueTransformer.class); + if (transformer == null) { + log.error("KeyValueTransformer not found in PluginContext."); + return null; + } + transformer.loadMapping(file); + // Perform the transformation + return transformer.transform(key, file); + } + +} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtCategory.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtCategory.java deleted file mode 100644 index f1f916fabdb..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtCategory.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.client.sn.TurMultiValue; -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; -import com.viglet.turing.sprinklr.client.service.folder.TurSprinklrFolderService; -import com.viglet.turing.sprinklr.client.service.token.TurSprinklrAccessToken; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -@Slf4j -public class TurSprinklrExtCategory implements TurSprinklrExtInterface { - - @Override - public Optional consume(TurSprinklrContext context) { - List categories = context.getSearchResult().getMappingDetails().getFirst().getMappedCategoryIds(); - TurSprinklrAccessToken turSprinklrAccessToken = context.getAccessToken(); - TurSprinklrFolderService turSprinklrFolderService = new TurSprinklrFolderService(turSprinklrAccessToken); - List categoryNames = new ArrayList<>(); - if (!categories.isEmpty() && turSprinklrAccessToken != null) { - categories.forEach(categoryId -> turSprinklrFolderService.getByCategoryId(categoryId) - .ifPresentOrElse(c -> categoryNames.add(c.getName()), - () -> categoryNames.add(categoryId))); - return Optional.of(new TurMultiValue(categoryNames)); - } else { - return Optional.empty(); - } - } -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtDescription.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtDescription.java deleted file mode 100644 index 372808069c0..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtDescription.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.commons.utils.TurCommonsUtils; -import com.viglet.turing.client.sn.TurMultiValue; -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; - -import java.util.Optional; - -public class TurSprinklrExtDescription implements TurSprinklrExtInterface { - - @Override - public Optional consume(TurSprinklrContext context) { - return Optional.of(TurMultiValue.singleItem( - TurCommonsUtils.html2Description( - context.getSearchResult().getContent().getMarkUpText(), - 300))); - - } -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtId.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtId.java deleted file mode 100644 index 049829f1972..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtId.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.client.sn.TurMultiValue; -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; - -import java.util.Optional; - -public class TurSprinklrExtId implements TurSprinklrExtInterface { - - @Override - public Optional consume(TurSprinklrContext context) { - return Optional.of(TurMultiValue.singleItem("sprinklr_" + context.getSearchResult().getId())); - } -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtInterface.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtInterface.java deleted file mode 100644 index 28fe0d21105..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtInterface.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.client.sn.TurMultiValue; -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; - -import java.util.Optional; - -public interface TurSprinklrExtInterface { - Optional consume(TurSprinklrContext context); -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtLocale.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtLocale.java deleted file mode 100644 index 4de7c66da89..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtLocale.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; - -import java.util.Locale; - -public class TurSprinklrExtLocale implements TurSprinklrExtLocaleInterface { - - @Override - public Locale consume(TurSprinklrContext context) { - return context.getSearchResult().getLocale(); - } -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtLocaleInterface.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtLocaleInterface.java deleted file mode 100644 index 4940b926345..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtLocaleInterface.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; - -import java.util.Locale; - -public interface TurSprinklrExtLocaleInterface { - Locale consume(TurSprinklrContext context); -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtModificationDate.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtModificationDate.java deleted file mode 100644 index 2a95150d8a4..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtModificationDate.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.client.sn.TurMultiValue; -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; -import lombok.extern.slf4j.Slf4j; - -import java.util.Optional; - -@Slf4j -public class TurSprinklrExtModificationDate implements TurSprinklrExtInterface { - @Override - public Optional consume(TurSprinklrContext context) { - return Optional.of(TurMultiValue.singleItem(context.getSearchResult().getModifiedTime())); - } -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtPublicationDate.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtPublicationDate.java deleted file mode 100644 index b49210e83cc..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtPublicationDate.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.client.sn.TurMultiValue; -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; -import lombok.extern.slf4j.Slf4j; - -import java.util.Optional; - -@Slf4j -public class TurSprinklrExtPublicationDate implements TurSprinklrExtInterface { - @Override - public Optional consume(TurSprinklrContext context) { - return Optional.of(TurMultiValue.singleItem(context.getSearchResult().getPublishingDate())); - } -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtTag.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtTag.java deleted file mode 100644 index a0600392821..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtTag.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.client.sn.TurMultiValue; -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; - -import java.util.List; -import java.util.Optional; - -public class TurSprinklrExtTag implements TurSprinklrExtInterface { - - @Override - public Optional consume(TurSprinklrContext context) { - List tags = context.getSearchResult().getTags(); - if (!tags.isEmpty()) { - return Optional.of(new TurMultiValue(tags)); - } - else { - return Optional.empty(); - } - } -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtText.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtText.java deleted file mode 100644 index af9fc974c0e..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtText.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.commons.utils.TurCommonsUtils; -import com.viglet.turing.client.sn.TurMultiValue; -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; - -import java.util.Optional; - -public class TurSprinklrExtText implements TurSprinklrExtInterface { - - @Override - public Optional consume(TurSprinklrContext context) { - return Optional.of( TurMultiValue.singleItem( - TurCommonsUtils.html2Text(context.getSearchResult().getContent().getMarkUpText()))); - } -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtTitle.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtTitle.java deleted file mode 100644 index 538d440b921..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtTitle.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.client.sn.TurMultiValue; -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; - -import java.util.Optional; - -public class TurSprinklrExtTitle implements TurSprinklrExtInterface { - - @Override - public Optional consume(TurSprinklrContext context) { - return Optional.of(TurMultiValue.singleItem(context.getSearchResult().getContent().getTitle())); - } -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtType.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtType.java deleted file mode 100644 index b5e1cf3ab48..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtType.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.client.sn.TurMultiValue; -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; - -import java.util.Optional; - -public class TurSprinklrExtType implements TurSprinklrExtInterface { - - @Override - public Optional consume(TurSprinklrContext context) { - return Optional.of(TurMultiValue.singleItem("%s:%s" - .formatted(context.getSearchResult().getContent().getContentType(), - context.getSearchResult().getContent().getContentSubType()))); - } -} diff --git a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtUrl.java b/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtUrl.java deleted file mode 100644 index 362e69a1ead..00000000000 --- a/turing-sprinklr/sprinklr-commons/src/main/java/com/viglet/turing/connector/sprinklr/ext/TurSprinklrExtUrl.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.viglet.turing.connector.sprinklr.ext; - -import com.viglet.turing.client.sn.TurMultiValue; -import com.viglet.turing.connector.sprinklr.TurSprinklrContext; -import com.viglet.turing.sprinklr.client.service.kb.response.TurSprinklrSearchResult; - -import java.util.Optional; - -public class TurSprinklrExtUrl implements TurSprinklrExtInterface { - - @Override - public Optional consume(TurSprinklrContext context) { - return Optional.of(Optional.ofNullable(context.getSearchResult()) - .map(TurSprinklrSearchResult::getMappingDetails) - .filter(turSprinklrMappings -> !turSprinklrMappings.isEmpty()) - .map(turSprinklrMappings -> - TurMultiValue.singleItem(turSprinklrMappings.getFirst().getCommunityPermalink())) - .orElse(TurMultiValue.empty())); - - } -} diff --git a/turing-sprinklr/sprinklr-java-sdk/pom.xml b/turing-sprinklr/sprinklr-java-sdk/pom.xml index 967bf8e914e..fb9767a7774 100644 --- a/turing-sprinklr/sprinklr-java-sdk/pom.xml +++ b/turing-sprinklr/sprinklr-java-sdk/pom.xml @@ -45,6 +45,10 @@ guava 33.3.1-jre + + com.viglet.turing + turing-commons + diff --git a/turing-sprinklr/sprinklr-java-sdk/src/main/java/com/viglet/turing/sprinklr/client/service/TurSprinklrService.java b/turing-sprinklr/sprinklr-java-sdk/src/main/java/com/viglet/turing/sprinklr/client/service/TurSprinklrService.java index 971fe696ba8..4bfecbfa50f 100644 --- a/turing-sprinklr/sprinklr-java-sdk/src/main/java/com/viglet/turing/sprinklr/client/service/TurSprinklrService.java +++ b/turing-sprinklr/sprinklr-java-sdk/src/main/java/com/viglet/turing/sprinklr/client/service/TurSprinklrService.java @@ -18,9 +18,12 @@ package com.viglet.turing.sprinklr.client.service; +import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.viglet.turing.sprinklr.client.service.token.TurSprinklrAccessToken; +import com.viglet.turing.sprinklr.client.service.token.TurSprinklrSecretKey; +import com.viglet.turing.sprinklr.client.service.token.TurSprinklrTokenService; import lombok.extern.slf4j.Slf4j; import okhttp3.*; @@ -50,7 +53,8 @@ public static R executeService(Class clazz, TurSprinklrAccessToken turSpr RequestBody requestBody) { log.info("Post Request: {}", endpoint); // Creates a client to send a request - return getResponse(clazz, getRequest(turSprinklrAccessToken, endpoint, requestBody)); + return getResponse(clazz, getRequest(turSprinklrAccessToken, endpoint, requestBody), turSprinklrAccessToken, + true); } private static Request getRequest(TurSprinklrAccessToken turSprinklrAccessToken, String endpoint, @@ -67,13 +71,33 @@ private static Request getRequest(TurSprinklrAccessToken turSprinklrAccessToken, return request; } - private static R getResponse(Class clazz, Request request) { + private static R getResponse(Class clazz, Request request, TurSprinklrAccessToken turSprinklrAccessToken, + boolean firstTime) { + String responseBody = null; try (Response response = new OkHttpClient().newBuilder().build().newCall(request).execute()) { - if (response.body() != null) { - return new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .readValue(response.body().string(), clazz); + log.info("HTTP Status Code: {}", response.code()); + if (response.code() == 200) { + responseBody = response.peekBody(500L).string(); + if (response.body() != null) { + return new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .readValue(response.body().string(), clazz); + } } + else if (firstTime) { + TurSprinklrTokenService turSprinklrTokenService = new TurSprinklrTokenService( + TurSprinklrSecretKey.builder() + .apiKey(turSprinklrAccessToken.getApiKey()) + .secretKey(turSprinklrAccessToken.getSecretKey()) + .environment(turSprinklrAccessToken.getEnvironment()) + .build() + ); + getResponse(clazz, request, turSprinklrTokenService.renewAccessToken(), false); + } + + } catch (JsonParseException e) { + log.error("Error parsing the response", e); + log.error("The body of the response is: {}", responseBody); } catch (IOException e) { log.error(e.getMessage(), e); } diff --git a/turing-sprinklr/sprinklr-java-sdk/src/main/java/com/viglet/turing/sprinklr/client/service/token/TurSprinklrAccessToken.java b/turing-sprinklr/sprinklr-java-sdk/src/main/java/com/viglet/turing/sprinklr/client/service/token/TurSprinklrAccessToken.java index 25392b95194..fbd5ec16d28 100644 --- a/turing-sprinklr/sprinklr-java-sdk/src/main/java/com/viglet/turing/sprinklr/client/service/token/TurSprinklrAccessToken.java +++ b/turing-sprinklr/sprinklr-java-sdk/src/main/java/com/viglet/turing/sprinklr/client/service/token/TurSprinklrAccessToken.java @@ -42,4 +42,5 @@ public class TurSprinklrAccessToken implements Serializable { private int expiresIn; private Date expirationDate; private String environment; + private String secretKey; } diff --git a/turing-sprinklr/sprinklr-java-sdk/src/main/java/com/viglet/turing/sprinklr/client/service/token/TurSprinklrTokenService.java b/turing-sprinklr/sprinklr-java-sdk/src/main/java/com/viglet/turing/sprinklr/client/service/token/TurSprinklrTokenService.java index 66716fefeb8..793de180339 100644 --- a/turing-sprinklr/sprinklr-java-sdk/src/main/java/com/viglet/turing/sprinklr/client/service/token/TurSprinklrTokenService.java +++ b/turing-sprinklr/sprinklr-java-sdk/src/main/java/com/viglet/turing/sprinklr/client/service/token/TurSprinklrTokenService.java @@ -57,6 +57,10 @@ public TurSprinklrAccessToken getAccessToken() { } } + public TurSprinklrAccessToken renewAccessToken() { + return serializeAccessToken(turSprinklrSecretKey); + } + private TurSprinklrAccessToken serializeAccessToken(TurSprinklrSecretKey turSprinklrSecretKey) { TurSprinklrAccessToken turSprinklrAccessToken = generateAccessToken(turSprinklrSecretKey); if (turSprinklrAccessToken != null) { @@ -100,6 +104,7 @@ private TurSprinklrAccessToken generateAccessToken(TurSprinklrSecretKey turSprin turSprinklrAccessToken.setApiKey(turSprinklrSecretKey.getApiKey()); turSprinklrAccessToken.setExpirationDate(getExprirationDate(turSprinklrAccessToken)); turSprinklrAccessToken.setEnvironment(turSprinklrSecretKey.getEnvironment()); + turSprinklrAccessToken.setSecretKey(turSprinklrSecretKey.getSecretKey()); return turSprinklrAccessToken; } catch (IOException e) { log.error(e.getMessage(), e); diff --git a/turing-ui/projects/console/src/sn/component/site/sn-site-ui-page.component.html b/turing-ui/projects/console/src/sn/component/site/sn-site-ui-page.component.html index c7994750c95..3877352c9c4 100644 --- a/turing-ui/projects/console/src/sn/component/site/sn-site-ui-page.component.html +++ b/turing-ui/projects/console/src/sn/component/site/sn-site-ui-page.component.html @@ -20,6 +20,17 @@

+
+ +

+ Exact match when using double quotes. Will use default field to execute the query. +

+
Wildcard @@ -226,6 +237,30 @@
What is the fields that will be used on search results display.
+
+
+ +
+
+ +

+ Exact match field when no field was specified in the query and it uses double quote. +

+
+
+
+
+ +
+
+ +

+ Default field when no field was specified in the query. +

+
+
diff --git a/turing-ui/projects/console/src/sn/model/sn-site.model.ts b/turing-ui/projects/console/src/sn/model/sn-site.model.ts index f6c88a7f455..0a22dea9feb 100644 --- a/turing-ui/projects/console/src/sn/model/sn-site.model.ts +++ b/turing-ui/projects/console/src/sn/model/sn-site.model.ts @@ -7,6 +7,8 @@ export interface TurSNSite { id: string; name: string; description: string; + exactMatchField: string; + defaultField: string; defaultTitleField: string; defaultDescriptionField: string; defaultTextField: string; @@ -32,4 +34,5 @@ export interface TurSNSite { facetSort: TurSNSiteFacetSortEnum; wildcardNoResults: number; wildcardAlways: number; + exactMatch: number; } diff --git a/turing-web-crawler/.gitignore b/turing-web-crawler/.gitignore index b83d22266ac..047f8eeee6d 100644 --- a/turing-web-crawler/.gitignore +++ b/turing-web-crawler/.gitignore @@ -1 +1,2 @@ /target/ +/wc-plugin/target/ diff --git a/turing-web-crawler/pom.xml b/turing-web-crawler/pom.xml index 83013f608f6..7795c95a672 100644 --- a/turing-web-crawler/pom.xml +++ b/turing-web-crawler/pom.xml @@ -40,6 +40,7 @@ wc-commons wc-app wc-sample + wc-plugin diff --git a/turing-web-crawler/wc-app/pom.xml b/turing-web-crawler/wc-app/pom.xml index 63e28338a0c..0cadb444d30 100644 --- a/turing-web-crawler/wc-app/pom.xml +++ b/turing-web-crawler/wc-app/pom.xml @@ -65,7 +65,7 @@ io.swagger.core.v3 swagger-annotations-jakarta - 2.2.26 + 2.2.27 compile diff --git a/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/TurWCProcess.java b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/TurWCProcess.java index 7b259746227..94d8888ba81 100644 --- a/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/TurWCProcess.java +++ b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/TurWCProcess.java @@ -15,10 +15,7 @@ import com.viglet.turing.connector.webcrawler.commons.ext.TurWCExtLocaleInterface; import com.viglet.turing.connector.webcrawler.persistence.model.TurWCAttributeMapping; import com.viglet.turing.connector.webcrawler.persistence.model.TurWCSource; -import com.viglet.turing.connector.webcrawler.persistence.repository.TurWCAllowUrlRepository; -import com.viglet.turing.connector.webcrawler.persistence.repository.TurWCAttributeMappingRepository; -import com.viglet.turing.connector.webcrawler.persistence.repository.TurWCFileExtensionRepository; -import com.viglet.turing.connector.webcrawler.persistence.repository.TurWCNotAllowUrlRepository; +import com.viglet.turing.connector.webcrawler.persistence.repository.*; import generator.RandomUserAgentGenerator; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -45,14 +42,21 @@ public class TurWCProcess { public static final String JAVASCRIPT = "javascript:"; public static final String A_HREF = "a[href]"; public static final String ABS_HREF = "abs:href"; + public static final String WILD_CARD = "*"; private final String turingUrl; private final String turingApiKey; + private final List startingPoints = new ArrayList<>(); + private final List allowUrls = new ArrayList<>(); + private final List allowStartsWithUrls = new ArrayList<>(); private final List notAllowUrls = new ArrayList<>(); + private final List notAllowStartsWithUrls = new ArrayList<>(); private final List notAllowExtensions = new ArrayList<>(); + private final TurWCStartingPointRepository turWCStartingPointsRepository; private TurSNJobItems turSNJobItems = new TurSNJobItems(); private final String userAgent = RandomUserAgentGenerator.getNextNonMobile(); private final Set visitedLinks = new HashSet<>(); - private final Queue remainingLinks = new LinkedList<>(); + private final Set indexedLinks = new HashSet<>(); + private final Queue queueLinks = new LinkedList<>(); private String website; private Collection snSites; private final int timeout; @@ -74,7 +78,7 @@ public TurWCProcess(@Value("${turing.url}") String turingUrl, TurWCAllowUrlRepository turWCAllowUrlRepository, TurWCNotAllowUrlRepository turWCNotAllowUrlRepository, TurWCFileExtensionRepository turWCFileExtensionRepository, - TurWCAttributeMappingRepository turWCAttributeMappingRepository) { + TurWCAttributeMappingRepository turWCAttributeMappingRepository, TurWCStartingPointRepository turWCStartingPointsRepository) { this.turingUrl = turingUrl; this.turingApiKey = turingApiKey; this.timeout = timeout; @@ -84,26 +88,45 @@ public TurWCProcess(@Value("${turing.url}") String turingUrl, this.turWCNotAllowUrlRepository = turWCNotAllowUrlRepository; this.turWCFileExtensionRepository = turWCFileExtensionRepository; this.turWCAttributeMappingRepository = turWCAttributeMappingRepository; + this.turWCStartingPointsRepository = turWCStartingPointsRepository; } public void start(TurWCSource turWCSource) { reset(); - turWCFileExtensionRepository.findByTurWCSource(turWCSource).ifPresent(source -> source.forEach(turWCFileExtension -> - this.notAllowExtensions.add(turWCFileExtension.getExtension()))); - turWCNotAllowUrlRepository.findByTurWCSource(turWCSource).ifPresent(source -> source.forEach(turWCNotAllowUrl -> - this.notAllowUrls.add(turWCNotAllowUrl.getUrl()))); - + turWCFileExtensionRepository.findByTurWCSource(turWCSource).ifPresent(source -> + source.forEach(turWCFileExtension -> + this.notAllowExtensions.add(turWCFileExtension.getExtension()))); + turWCNotAllowUrlRepository.findByTurWCSource(turWCSource).ifPresent(source -> + source.forEach(turWCNotAllowUrl -> { + if (turWCNotAllowUrl.getUrl().trim().endsWith(WILD_CARD)) { + this.notAllowStartsWithUrls.add(StringUtils.chop(turWCNotAllowUrl.getUrl())); + } else { + this.notAllowUrls.add(turWCNotAllowUrl.getUrl()); + } + } + )); + turWCAllowUrlRepository.findByTurWCSource(turWCSource).ifPresent(source -> + source.forEach(turWCAllowUrl -> { + if (turWCAllowUrl.getUrl().trim().endsWith(WILD_CARD)) { + this.allowStartsWithUrls.add(StringUtils.chop(turWCAllowUrl.getUrl().trim())); + } else { + this.allowUrls.add(turWCAllowUrl.getUrl()); + } + } + )); + turWCStartingPointsRepository.findByTurWCSource(turWCSource).ifPresent(source -> + source.forEach(turWCStartingPoint -> + this.startingPoints.add(turWCStartingPoint.getUrl()) + )); this.website = turWCSource.getUrl(); this.snSites = turWCSource.getTurSNSites(); this.username = turWCSource.getUsername(); this.password = turWCSource.getPassword(); log.info("User Agent: {}", userAgent); - turWCAllowUrlRepository - .findByTurWCSource(turWCSource).ifPresent(source -> - source.forEach(turWCAllowUrl -> { - remainingLinks.add(this.website + turWCAllowUrl.getUrl()); - getPagesFromQueue(turWCSource); - })); + startingPoints.forEach(url -> { + queueLinks.offer(this.website + url); + getPagesFromQueue(turWCSource); + }); if (turSNJobItems.size() > 0) { sendToTuring(); getInfoQueue(); @@ -118,12 +141,13 @@ private void reset() { private void getInfoQueue() { log.info("Total Job Item: {}", Iterators.size(turSNJobItems.iterator())); log.info("Total Visited Links: {}", (long) visitedLinks.size()); - log.info("Queue Size: {}", (long) remainingLinks.size()); + log.info("Total Indexed Links: {}", (long) indexedLinks.size()); + log.info("Queue Size: {}", (long) queueLinks.size()); } public void getPagesFromQueue(TurWCSource turWCSource) { - while (!remainingLinks.isEmpty()) { - String url = remainingLinks.poll(); + while (!queueLinks.isEmpty()) { + String url = queueLinks.poll(); getPage(turWCSource, url); sendToTuringWhenMaxSize(); getInfoQueue(); @@ -135,36 +159,52 @@ public TurSNJobItem getPage(TurWCSource turWCSource, String url) { log.info("{}: {}", url, turWCSource.getTurSNSites()); Document document = getHTML(url); getPageLinks(document); - return addTurSNJobItems(turWCSource, document, url); + String pageUrl = getPageUrl(url); + if (canBeIndexed(pageUrl)) { + indexedLinks.add(pageUrl); + return addTurSNJobItems(turWCSource, document, url); + } else { + log.debug("Ignored: {}", url); + } } catch (IOException e) { - log.error(e.getMessage()); + log.error(e.getMessage(), e); } - return new TurSNJobItem(); } - private TurSNJobItem addTurSNJobItems(TurWCSource turWCSource, Document document, String url) { - TurSNJobItem turSNJobItem = new TurSNJobItem(TurSNJobAction.CREATE, new ArrayList<>(snSites), - getLocale(turWCSource, document, url), - getJobItemAttributes(turWCSource, document, url)); - turSNJobItems.add(turSNJobItem); - return turSNJobItem; - } - private void getPageLinks(Document document) { document.select(A_HREF).forEach(page -> addPageToQueue(getPageUrl(page.attr(ABS_HREF)))); } private void addPageToQueue(String pageUrl) { - if (canBeIndexed(pageUrl)) { - if (visitedLinks.add(pageUrl) && !remainingLinks.offer(pageUrl)) { - log.error("Item didn't add to queue: {}", pageUrl); - } - } else { - log.debug("Ignored: {}", pageUrl); + if (canBeAddToQueue(pageUrl) && visitedLinks.add(pageUrl) && !queueLinks.offer(pageUrl)) { + log.error("Item didn't add to queue: {}", pageUrl); } } + private boolean isValidToAddQueue(String pageUrl) { + return isNotMailUrl(pageUrl) + && isNotTelUrl(pageUrl) + && !StringUtils.equalsAny(pageUrl, queueLinks.toArray(new String[0])) + && !isSharpUrl(pageUrl) && !isPagination(pageUrl) && !isJavascriptUrl(pageUrl) + && pageUrl.startsWith(this.website) + && ( + StringUtils.startsWithAny(getRelativePageUrl(pageUrl), allowStartsWithUrls.toArray(new String[0])) + || StringUtils.equalsAny(getRelativePageUrl(pageUrl), allowUrls.toArray(new String[0])) + ) + && !StringUtils.startsWithAny(getRelativePageUrl(pageUrl), notAllowStartsWithUrls.toArray(new String[0])) + && !StringUtils.equalsAny(getRelativePageUrl(pageUrl), notAllowUrls.toArray(new String[0])) + && !StringUtils.endsWithAny(pageUrl, notAllowExtensions.toArray(new String[0])); + } + + private TurSNJobItem addTurSNJobItems(TurWCSource turWCSource, Document document, String url) { + TurSNJobItem turSNJobItem = new TurSNJobItem(TurSNJobAction.CREATE, new ArrayList<>(snSites), + getLocale(turWCSource, document, url), + getJobItemAttributes(turWCSource, document, url)); + turSNJobItems.add(turSNJobItem); + return turSNJobItem; + } + private void sendToTuringWhenMaxSize() { if (turSNJobItems.size() >= jobSize) { sendToTuring(); @@ -235,7 +275,6 @@ private static void addItemToArray(Map attributes, String attrib .stream().map(String.class::cast).toList()); attributeValues.add(attributeValue); attributes.put(attributeName, attributeValues); - } private void addFirstItemToAttribute(String attributeName, @@ -244,7 +283,6 @@ private void addFirstItemToAttribute(String attributeName, attributes.put(attributeName, attributeValue); } - private void sendToTuring() { if (log.isDebugEnabled()) { for (TurSNJobItem turSNJobItem : turSNJobItems) { @@ -288,16 +326,16 @@ public TurWCContext getTurWCContext(Document document, String url) { } private boolean canBeIndexed(String pageUrl) { - return !isSharpUrl(pageUrl) && !isPagination(pageUrl) && !isJavascriptUrl(pageUrl) - && pageUrl.startsWith(this.website) - && !StringUtils.startsWithAny(getRelativePageUrl(pageUrl), - notAllowUrls.toArray(new String[0])) - && !StringUtils.endsWithAny(pageUrl, - notAllowExtensions.toArray(new String[0])) - && !StringUtils.equalsAny(pageUrl, - visitedLinks.toArray(new String[0])); + return isValidToAddQueue(pageUrl) + && !StringUtils.equalsAny(pageUrl, indexedLinks.toArray(new String[0])); + } + + private boolean canBeAddToQueue(String pageUrl) { + return isValidToAddQueue(pageUrl) + && !StringUtils.equalsAny(pageUrl, visitedLinks.toArray(new String[0])); } + private static boolean isJavascriptUrl(String pageUrl) { return pageUrl.contains(JAVASCRIPT); } @@ -382,4 +420,4 @@ private String getUrlWithoutParameters(String url) { return url; } } -} +} \ No newline at end of file diff --git a/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/TurWCScheduledTasks.java b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/TurWCScheduledTasks.java index 37f2ed2a249..4fac8759285 100644 --- a/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/TurWCScheduledTasks.java +++ b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/TurWCScheduledTasks.java @@ -1,6 +1,7 @@ package com.viglet.turing.connector.webcrawler; import com.google.inject.Inject; +import com.viglet.turing.connector.webcrawler.persistence.repository.TurWCConfigVarRepository; import com.viglet.turing.connector.webcrawler.persistence.repository.TurWCSourceRepository; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Scheduled; @@ -14,18 +15,28 @@ @Component public class TurWCScheduledTasks { private final TurWCSourceRepository turWCSourceRepository; + private final TurWCConfigVarRepository turWCConfigVarRepository; private final TurWCProcess turWCProcess; private final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); + public static final String FIRST_TIME = "FIRST_TIME"; @Inject - public TurWCScheduledTasks(TurWCSourceRepository turWCSourceRepository, TurWCProcess turWCProcess) { + public TurWCScheduledTasks(TurWCSourceRepository turWCSourceRepository, + TurWCConfigVarRepository turWCConfigVarRepository, + TurWCProcess turWCProcess) { this.turWCSourceRepository = turWCSourceRepository; + this.turWCConfigVarRepository = turWCConfigVarRepository; this.turWCProcess = turWCProcess; } @Scheduled(fixedDelay = 60, timeUnit = TimeUnit.MINUTES) public void executeWebCrawler() { log.info("The time is now {}", dateFormat.format(new Date())); - turWCSourceRepository.findAll().forEach(turWCProcess::start); + if (turWCConfigVarRepository.findById(FIRST_TIME).isEmpty()) { + log.info("This is the first time, waiting next schedule."); + } else { + log.info("Starting indexing"); + turWCSourceRepository.findAll().forEach(turWCProcess::start); + } } } diff --git a/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/export/TurWCExchangeProcess.java b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/export/TurWCExchangeProcess.java index b60b8118d03..0bcc3e76941 100644 --- a/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/export/TurWCExchangeProcess.java +++ b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/export/TurWCExchangeProcess.java @@ -31,7 +31,9 @@ public class TurWCExchangeProcess { private static final String EXPORT_FILE = "export.json"; private final TurWCSourceRepository turWCSourceRepository; + private final TurWCAllowUrlRepository turWCAllowUrlRepository; + private final TurWCStartingPointRepository turWCStartingPointRepository; private final TurWCNotAllowUrlRepository turWCNotAllowUrlRepository; private final TurWCFileExtensionRepository turWCFileExtensionRepository; private final TurWCAttributeMappingRepository turWCAttributeMappingRepository; @@ -39,11 +41,13 @@ public class TurWCExchangeProcess { @Inject public TurWCExchangeProcess(TurWCSourceRepository turWCSourceRepository, TurWCAllowUrlRepository turWCAllowUrlRepository, + TurWCStartingPointRepository turWCStartingPointRepository, TurWCNotAllowUrlRepository turWCNotAllowUrlRepository, TurWCFileExtensionRepository turWCFileExtensionRepository, TurWCAttributeMappingRepository turWCAttributeMappingRepository) { this.turWCSourceRepository = turWCSourceRepository; this.turWCAllowUrlRepository = turWCAllowUrlRepository; + this.turWCStartingPointRepository = turWCStartingPointRepository; this.turWCNotAllowUrlRepository = turWCNotAllowUrlRepository; this.turWCFileExtensionRepository = turWCFileExtensionRepository; this.turWCAttributeMappingRepository = turWCAttributeMappingRepository; @@ -72,7 +76,6 @@ public StreamingResponseBody exportObject(HttpServletResponse response) { List turWCSources = turWCSourceRepository.findAll(); - File exportDir = new File(tmpDir.getAbsolutePath().concat(File.separator + folderName)); File exportFile = new File(exportDir.getAbsolutePath().concat(File.separator + EXPORT_FILE)); try { @@ -176,7 +179,7 @@ public void importWCSource(TurWCExchange turWCExchange) { for (TurWCSourceExchange turWCSourceExchange : turWCExchange.getSources()) { if (turWCSourceRepository.findById(turWCSourceExchange.getId()).isEmpty()) { TurWCSource turWCSource = TurWCSource.builder() - .id(turWCSourceExchange.getId()) + // .id(turWCSourceExchange.getId()) .url(turWCSourceExchange.getUrl()) .username(turWCSourceExchange.getUsername()) .password(turWCSourceExchange.getPassword()) @@ -187,6 +190,11 @@ public void importWCSource(TurWCExchange turWCExchange) { turWCSourceRepository.save(turWCSource); + turWCSourceExchange.getStartingPoints().forEach(url -> + turWCStartingPointRepository.save(TurWCStartingPoint.builder() + .url(url) + .turWCSource(turWCSource) + .build())); turWCSourceExchange.getAllowUrls().forEach(url -> turWCAllowUrlRepository.save(TurWCAllowUrl.builder() .url(url) diff --git a/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/export/bean/TurWCSourceExchange.java b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/export/bean/TurWCSourceExchange.java index 993f6a670d5..fa3f6e3debe 100644 --- a/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/export/bean/TurWCSourceExchange.java +++ b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/export/bean/TurWCSourceExchange.java @@ -21,6 +21,8 @@ public class TurWCSourceExchange { private String username; private String password; @Builder.Default + private Collection startingPoints = new HashSet<>(); + @Builder.Default private Collection allowUrls = new HashSet<>(); @Builder.Default private Collection notAllowUrls = new HashSet<>(); diff --git a/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/persistence/model/TurWCSource.java b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/persistence/model/TurWCSource.java index 7abecc9fd3a..f6f51c19a75 100644 --- a/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/persistence/model/TurWCSource.java +++ b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/persistence/model/TurWCSource.java @@ -54,6 +54,12 @@ public class TurWCSource implements Serializable { @Column(name = SN_SITE, nullable = false) private Collection turSNSites = new HashSet<>(); + @Builder.Default + @OneToMany(mappedBy = TUR_WC_SOURCE, orphanRemoval = true, fetch = FetchType.LAZY) + @Cascade({org.hibernate.annotations.CascadeType.ALL, org.hibernate.annotations.CascadeType.DELETE_ORPHAN}) + @OnDelete(action = OnDeleteAction.CASCADE) + private Collection startingPoints = new HashSet<>(); + @Builder.Default @OneToMany(mappedBy = TUR_WC_SOURCE, orphanRemoval = true, fetch = FetchType.LAZY) @Cascade({org.hibernate.annotations.CascadeType.ALL, org.hibernate.annotations.CascadeType.DELETE_ORPHAN}) @@ -78,6 +84,14 @@ public class TurWCSource implements Serializable { @OnDelete(action = OnDeleteAction.CASCADE) private Collection attributeMappings = new HashSet<>(); + public void setStartingPoints(Collection startingPoints) { + this.startingPoints.clear(); + if (startingPoints != null) { + this.startingPoints.addAll(startingPoints); + } + } + + public void setAllowUrls(Collection allowUrls) { this.allowUrls.clear(); if (allowUrls != null) { diff --git a/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/persistence/model/TurWCStartingPoint.java b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/persistence/model/TurWCStartingPoint.java new file mode 100644 index 00000000000..441e39ccacd --- /dev/null +++ b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/persistence/model/TurWCStartingPoint.java @@ -0,0 +1,37 @@ +package com.viglet.turing.connector.webcrawler.persistence.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import jakarta.persistence.Entity; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; + +import java.io.Serial; +import java.io.Serializable; + +@SuperBuilder +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Entity +@JsonIgnoreProperties({ "turWCSource" }) +public class TurWCStartingPoint extends TurWCUrl implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + // bi-directional many-to-one association to TurWCSource + @ManyToOne + @JoinColumn(name = "wc_source_id", nullable = false) + private TurWCSource turWCSource; + + public TurWCStartingPoint(String url, TurWCSource turWCSource) { + this.url = url; + this.turWCSource = turWCSource; + } +} diff --git a/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/persistence/repository/TurWCStartingPointRepository.java b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/persistence/repository/TurWCStartingPointRepository.java new file mode 100644 index 00000000000..56339c0daee --- /dev/null +++ b/turing-web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/persistence/repository/TurWCStartingPointRepository.java @@ -0,0 +1,12 @@ +package com.viglet.turing.connector.webcrawler.persistence.repository; + +import com.viglet.turing.connector.webcrawler.persistence.model.TurWCSource; +import com.viglet.turing.connector.webcrawler.persistence.model.TurWCStartingPoint; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface TurWCStartingPointRepository extends JpaRepository { + Optional> findByTurWCSource(TurWCSource turWCSource); +} diff --git a/turing-web-crawler/wc-plugin/pom.xml b/turing-web-crawler/wc-plugin/pom.xml new file mode 100644 index 00000000000..72481782751 --- /dev/null +++ b/turing-web-crawler/wc-plugin/pom.xml @@ -0,0 +1,140 @@ + + + 4.0.0 + + turing-web-crawler + com.viglet.turing + ${revision} + + + wc-plugin + Turing Web Crawler - Connector Plugin + + + UTF-8 + UTF-8 + 21 + 21 + 21 + + + + + ch.qos.logback + logback-classic + + + com.h2database + h2 + + + org.projectlombok + lombok + true + + + com.viglet.turing + turing-spring + ${revision} + + + com.google.inject + guice + 7.0.0 + compile + + + com.fasterxml.jackson.datatype + jackson-datatype-hibernate5-jakarta + + + com.viglet.turing + turing-java-sdk + + + com.viglet.turing + wc-commons + ${revision} + + + com.sezinkarli + random-user-agent-generator + 1.3 + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-starter-test + test + + + com.viglet.turing + connector-commons + 0.3.9 + compile + + + + + + org.springframework.boot + spring-boot-dependencies + ${spring-boot.version} + pom + import + + + + + turing-web-crawler-plugin + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + true + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + + shade + + + turing-web-crawler-plugin + true + + + + *:* + + + com.viglet.turing:wc-commons:* + + + + + com.viglet.turing.connector.plugin.webcrawler.TurWCPlugin + + + + + + + + + diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/TurWCPlugin.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/TurWCPlugin.java new file mode 100644 index 00000000000..78e85bb0753 --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/TurWCPlugin.java @@ -0,0 +1,43 @@ +/* + * + * Copyright (C) 2016-2024 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.viglet.turing.connector.plugin.webcrawler; + +import com.google.inject.Inject; +import com.viglet.turing.client.sn.job.TurSNJobItem; +import com.viglet.turing.connector.plugin.TurConnectorPlugin; +import com.viglet.turing.connector.plugin.webcrawler.persistence.repository.TurWCSourceRepository; + +public class TurWCPlugin implements TurConnectorPlugin { + private final TurWCSourceRepository turWCSourceRepository; + private final TurWCPluginProcess turWCPluginProcess; + + @Inject + public TurWCPlugin(TurWCSourceRepository turWCSourceRepository, TurWCPluginProcess turWCPluginProcess) { + this.turWCSourceRepository = turWCSourceRepository; + this.turWCPluginProcess = turWCPluginProcess; + } + + public void init() { + turWCSourceRepository.findAll().forEach(turWCPluginProcess::start); + } + @Override + public TurSNJobItem getNext() { + return turWCPluginProcess.getNext(); + } +} \ No newline at end of file diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/TurWCPluginProcess.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/TurWCPluginProcess.java new file mode 100644 index 00000000000..4d5ba8b5936 --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/TurWCPluginProcess.java @@ -0,0 +1,376 @@ +package com.viglet.turing.connector.plugin.webcrawler; + +import com.google.inject.Inject; +import com.viglet.turing.client.sn.TurMultiValue; +import com.viglet.turing.client.sn.job.TurSNJobAction; +import com.viglet.turing.client.sn.job.TurSNJobItem; +import com.viglet.turing.client.sn.job.TurSNJobItems; +import com.viglet.turing.commons.cache.TurCustomClassCache; +import com.viglet.turing.connector.plugin.webcrawler.persistence.repository.*; +import com.viglet.turing.connector.webcrawler.commons.TurWCContext; +import com.viglet.turing.connector.webcrawler.commons.ext.TurWCExtInterface; +import com.viglet.turing.connector.webcrawler.commons.ext.TurWCExtLocaleInterface; +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCAttributeMapping; +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCSource; +import generator.RandomUserAgentGenerator; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.jsoup.Connection; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Entities; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.*; + +@Slf4j +@Component +public class TurWCPluginProcess { + public static final String MAILTO = "mailto"; + public static final String TEL = "tel:"; + public static final String JAVASCRIPT = "javascript:"; + public static final String A_HREF = "a[href]"; + public static final String ABS_HREF = "abs:href"; + public static final String WILD_CARD = "*"; + private final List startingPoints = new ArrayList<>(); + private final List allowUrls = new ArrayList<>(); + private final List allowStartsWithUrls = new ArrayList<>(); + private final List notAllowUrls = new ArrayList<>(); + private final List notAllowStartsWithUrls = new ArrayList<>(); + private final List notAllowExtensions = new ArrayList<>(); + private final TurWCStartingPointRepository turWCStartingPointsRepository; + private TurSNJobItems turSNJobItems = new TurSNJobItems(); + private final String userAgent = RandomUserAgentGenerator.getNextNonMobile(); + private final Set visitedLinks = new HashSet<>(); + private final Set indexedLinks = new HashSet<>(); + private final Queue queueLinks = new LinkedList<>(); + private String website; + private Collection snSites; + private final int timeout; + private final String referrer; + private String username; + private String password; + private final TurWCAllowUrlRepository turWCAllowUrlRepository; + private final TurWCNotAllowUrlRepository turWCNotAllowUrlRepository; + private final TurWCFileExtensionRepository turWCFileExtensionRepository; + private final TurWCAttributeMappingRepository turWCAttributeMappingRepository; + + @Inject + public TurWCPluginProcess(@Value("${turing.wc.timeout:5000}") int timeout, + @Value("${turing.wc.referrer:https://www.google.com}") String referrer, + TurWCAllowUrlRepository turWCAllowUrlRepository, + TurWCNotAllowUrlRepository turWCNotAllowUrlRepository, + TurWCFileExtensionRepository turWCFileExtensionRepository, + TurWCAttributeMappingRepository turWCAttributeMappingRepository, + TurWCStartingPointRepository turWCStartingPointsRepository) { + this.timeout = timeout; + this.referrer = referrer; + this.turWCAllowUrlRepository = turWCAllowUrlRepository; + this.turWCNotAllowUrlRepository = turWCNotAllowUrlRepository; + this.turWCFileExtensionRepository = turWCFileExtensionRepository; + this.turWCAttributeMappingRepository = turWCAttributeMappingRepository; + this.turWCStartingPointsRepository = turWCStartingPointsRepository; + } + + public void start(TurWCSource turWCSource) { + reset(); + turWCFileExtensionRepository.findByTurWCSource(turWCSource).ifPresent(source -> + source.forEach(turWCFileExtension -> + this.notAllowExtensions.add(turWCFileExtension.getExtension()))); + turWCNotAllowUrlRepository.findByTurWCSource(turWCSource).ifPresent(source -> + source.forEach(turWCNotAllowUrl -> { + if (turWCNotAllowUrl.getUrl().trim().endsWith(WILD_CARD)) { + this.notAllowStartsWithUrls.add(StringUtils.chop(turWCNotAllowUrl.getUrl())); + } else { + this.notAllowUrls.add(turWCNotAllowUrl.getUrl()); + } + } + )); + turWCAllowUrlRepository.findByTurWCSource(turWCSource).ifPresent(source -> + source.forEach(turWCAllowUrl -> { + if (turWCAllowUrl.getUrl().trim().endsWith(WILD_CARD)) { + this.allowStartsWithUrls.add(StringUtils.chop(turWCAllowUrl.getUrl().trim())); + } else { + this.allowUrls.add(turWCAllowUrl.getUrl()); + } + } + )); + turWCStartingPointsRepository.findByTurWCSource(turWCSource).ifPresent(source -> + source.forEach(turWCStartingPoint -> + this.startingPoints.add(turWCStartingPoint.getUrl()) + )); + this.website = turWCSource.getUrl(); + this.snSites = turWCSource.getTurSNSites(); + this.username = turWCSource.getUsername(); + this.password = turWCSource.getPassword(); + log.info("User Agent: {}", userAgent); + startingPoints.forEach(url -> { + queueLinks.offer(this.website + url); + getPagesFromQueue(turWCSource); + }); + } + + private void reset() { + turSNJobItems = new TurSNJobItems(); + visitedLinks.clear(); + } + + + public void getPagesFromQueue(TurWCSource turWCSource) { + while (!queueLinks.isEmpty()) { + String url = queueLinks.poll(); + getPage(turWCSource, url); + } + } + + public void getPage(TurWCSource turWCSource, String url) { + try { + log.info("{}: {}", url, turWCSource.getTurSNSites()); + Document document = getHTML(url); + getPageLinks(document); + String pageUrl = getPageUrl(url); + if (canBeIndexed(pageUrl)) { + indexedLinks.add(pageUrl); + addTurSNJobItems(turWCSource, document, url); + return; + } else { + log.debug("Ignored: {}", url); + } + } catch (IOException e) { + log.error(e.getMessage(), e); + } + new TurSNJobItem(); + } + + public TurSNJobItem getNext() { + return turSNJobItems.iterator().next(); + } + private void getPageLinks(Document document) { + document.select(A_HREF).forEach(page -> addPageToQueue(getPageUrl(page.attr(ABS_HREF)))); + } + + private void addPageToQueue(String pageUrl) { + if (canBeAddToQueue(pageUrl) && visitedLinks.add(pageUrl) && !queueLinks.offer(pageUrl)) { + log.error("Item didn't add to queue: {}", pageUrl); + } + } + + private boolean isValidToAddQueue(String pageUrl) { + return isNotMailUrl(pageUrl) + && isNotTelUrl(pageUrl) + && !StringUtils.equalsAny(pageUrl, queueLinks.toArray(new String[0])) + && !isSharpUrl(pageUrl) && !isPagination(pageUrl) && !isJavascriptUrl(pageUrl) + && pageUrl.startsWith(this.website) + && ( + StringUtils.startsWithAny(getRelativePageUrl(pageUrl), allowStartsWithUrls.toArray(new String[0])) + || StringUtils.equalsAny(getRelativePageUrl(pageUrl), allowUrls.toArray(new String[0])) + ) + && !StringUtils.startsWithAny(getRelativePageUrl(pageUrl), notAllowStartsWithUrls.toArray(new String[0])) + && !StringUtils.equalsAny(getRelativePageUrl(pageUrl), notAllowUrls.toArray(new String[0])) + && !StringUtils.endsWithAny(pageUrl, notAllowExtensions.toArray(new String[0])); + } + + private void addTurSNJobItems(TurWCSource turWCSource, Document document, String url) { + TurSNJobItem turSNJobItem = new TurSNJobItem(TurSNJobAction.CREATE, new ArrayList<>(snSites), + getLocale(turWCSource, document, url), + getJobItemAttributes(turWCSource, document, url)); + turSNJobItems.add(turSNJobItem); + } + + public Map getJobItemAttributes(TurWCSource turWCSource, Document document, String url) { + Map turSNJobItemAttributes = new HashMap<>(); + turWCAttributeMappingRepository.findByTurWCSource(turWCSource).ifPresent(source -> + source.forEach(turWCCustomClass -> + Optional.ofNullable(turWCCustomClass.getText()).ifPresentOrElse(text -> + usesText(turWCCustomClass, text, turSNJobItemAttributes) + , () -> { + if (!StringUtils.isEmpty(turWCCustomClass.getClassName())) + usesCustomClass(document, url, turWCCustomClass, turSNJobItemAttributes); + } + ))); + return turSNJobItemAttributes; + } + + private void usesCustomClass(Document document, String url, TurWCAttributeMapping turWCCustomClass, + Map turSNJobItemAttributes) { + getCustomClass(document, url, turWCCustomClass) + .ifPresent(turMultiValue -> turMultiValue.forEach(attributeValue -> { + if (!StringUtils.isBlank(attributeValue)) { + if (turSNJobItemAttributes.containsKey(turWCCustomClass.getName())) { + addItemInExistingAttribute(attributeValue, + turSNJobItemAttributes, turWCCustomClass.getName()); + } else { + addFirstItemToAttribute(turWCCustomClass.getName(), + attributeValue, turSNJobItemAttributes); + } + } + })); + } + + private static void usesText(TurWCAttributeMapping turWCCustomClass, String text, + Map turSNJobItemAttributes) { + turSNJobItemAttributes.put(turWCCustomClass.getName(), text); + } + + private Optional getCustomClass(Document document, String url, + TurWCAttributeMapping turWCAttributeMapping) { + return TurCustomClassCache.getCustomClassMap(turWCAttributeMapping.getClassName()) + .flatMap(classInstance -> ((TurWCExtInterface) classInstance) + .consume(getTurWCContext(document, url))); + } + + private static void addItemInExistingAttribute(String attributeValue, + Map attributes, + String attributeName) { + if (attributes.get(attributeName) instanceof ArrayList) + addItemToArray(attributes, attributeName, attributeValue); + else convertAttributeSingleValueToArray(attributes, attributeName, attributeValue); + } + + private static void convertAttributeSingleValueToArray(Map attributes, + String attributeName, String attributeValue) { + List attributeValues = new ArrayList<>(); + attributeValues.add(attributes.get(attributeName)); + attributeValues.add(attributeValue); + attributes.put(attributeName, attributeValues); + } + + private static void addItemToArray(Map attributes, String attributeName, String attributeValue) { + List attributeValues = new ArrayList<>(((List) attributes.get(attributeName)) + .stream().map(String.class::cast).toList()); + attributeValues.add(attributeValue); + attributes.put(attributeName, attributeValues); + } + + private void addFirstItemToAttribute(String attributeName, + String attributeValue, + Map attributes) { + attributes.put(attributeName, attributeValue); + } + + public Locale getLocale(TurWCSource turWCSource, Document document, String url) { + + return Optional.ofNullable(turWCSource.getLocale()) + .orElseGet(() -> { + if (!StringUtils.isEmpty(turWCSource.getLocaleClass())) { + return TurCustomClassCache.getCustomClassMap(turWCSource.getLocaleClass()) + .map(classInstance -> ((TurWCExtLocaleInterface) classInstance) + .consume(getTurWCContext(document, url))) + .orElse(Locale.US); + + } + return Locale.US; + }); + } + + public TurWCContext getTurWCContext(Document document, String url) { + return TurWCContext.builder() + .document(document) + .url(url) + .timeout(timeout) + .userAgent(userAgent) + .referrer(referrer) + .build(); + } + + private boolean canBeIndexed(String pageUrl) { + return isValidToAddQueue(pageUrl) + && !StringUtils.equalsAny(pageUrl, indexedLinks.toArray(new String[0])); + } + + private boolean canBeAddToQueue(String pageUrl) { + return isValidToAddQueue(pageUrl) + && !StringUtils.equalsAny(pageUrl, visitedLinks.toArray(new String[0])); + } + + + private static boolean isJavascriptUrl(String pageUrl) { + return pageUrl.contains(JAVASCRIPT); + } + + private String getPageUrl(String attr) { + String pageUrl = getUrlWithoutParameters(!isHttpUrl(attr) + && isNotMailUrl(attr) && isNotTelUrl(attr) ? this.website + attr : attr); + String pageUrlNormalized = pageUrl.endsWith("/") ? removeLastChar(pageUrl) : pageUrl; + if (isNotMailUrl(attr) && isNotTelUrl(attr)) { + try { + return URI.create(pageUrlNormalized).normalize().toString(); + } catch (IllegalArgumentException ignored) { + // No error + } + } + return pageUrlNormalized; + } + + private static String removeLastChar(String pageUrl) { + return pageUrl.substring(0, pageUrl.length() - 1); + } + + private Document getHTML(String url) throws IOException { + + Connection connection = Jsoup.connect(url) + .userAgent(userAgent) + .referrer(referrer) + .timeout(timeout); + if (isBasicAuth()) { + connection.header("Authorization", "Basic " + getBasicAuth()); + } + Document document = connection.get(); + + document.outputSettings().escapeMode(Entities.EscapeMode.xhtml); + document.outputSettings().charset(StandardCharsets.ISO_8859_1); + document.charset(StandardCharsets.ISO_8859_1); + return document; + } + + private String getBasicAuth() { + String authString = this.username + ":" + this.password; + return Base64.getEncoder().encodeToString(authString.getBytes()); + } + + private boolean isBasicAuth() { + return this.username != null; + } + + private String getRelativePageUrl(String pageUrl) { + return pageUrl.replaceAll(this.website, ""); + } + + private static boolean isPagination(String pageUrl) { + return pageUrl.contains("/page/"); + } + + private static boolean isSharpUrl(String attr) { + return attr.contains("#"); + } + + private static boolean isHttpUrl(String attr) { + return attr.toLowerCase().startsWith("http"); + } + + private static boolean isNotMailUrl(String attr) { + return !attr.toLowerCase().startsWith(MAILTO); + } + + private static boolean isNotTelUrl(String attr) { + return !attr.toLowerCase().startsWith(TEL); + } + + private String getUrlWithoutParameters(String url) { + try { + URI uri = new URI(url); + return new URI(uri.getScheme(), + uri.getAuthority(), + uri.getPath(), + null, + uri.getFragment()).toString(); + } catch (URISyntaxException e) { + return url; + } + } +} \ No newline at end of file diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCAllowUrl.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCAllowUrl.java new file mode 100644 index 00000000000..76303054c70 --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCAllowUrl.java @@ -0,0 +1,37 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import jakarta.persistence.Entity; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; + +import java.io.Serial; +import java.io.Serializable; + +@SuperBuilder +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Entity +@JsonIgnoreProperties({ "turWCSource" }) +public class TurWCAllowUrl extends TurWCUrl implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + // bi-directional many-to-one association to TurWCSource + @ManyToOne + @JoinColumn(name = "wc_source_id", nullable = false) + private TurWCSource turWCSource; + + public TurWCAllowUrl(String url, TurWCSource turWCSource) { + this.url = url; + this.turWCSource = turWCSource; + } +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCAttributeMapping.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCAttributeMapping.java new file mode 100644 index 00000000000..e9bfe0e27d9 --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCAttributeMapping.java @@ -0,0 +1,50 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.viglet.turing.spring.jpa.TurUuid; +import jakarta.persistence.*; +import lombok.*; + +import java.io.Serial; +import java.io.Serializable; + + +@Builder(toBuilder = true) +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Entity +@Table(name = "wc_attribute_mapping") +@JsonIgnoreProperties({ "turWCSource" }) +public class TurWCAttributeMapping implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + @Id + @TurUuid + @Column(name = "id", nullable = false) + private String id; + private String name; + private String className; + private String text; + + @ManyToOne + @JoinColumn(name = "ws_source_id", nullable = false) + private TurWCSource turWCSource; + + public TurWCAttributeMapping(String name, Class className, TurWCSource turWCSource) { + this.name = name; + this.className = className.getName(); + this.text = null; + this.turWCSource = turWCSource; + } + + public TurWCAttributeMapping(String name, String text, TurWCSource turWCSource) { + this.name = name; + this.className = null; + this.text = text; + this.turWCSource = turWCSource; + } +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCFileExtension.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCFileExtension.java new file mode 100644 index 00000000000..dfe05b59563 --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCFileExtension.java @@ -0,0 +1,40 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.viglet.turing.spring.jpa.TurUuid; +import jakarta.persistence.*; +import lombok.*; + +import java.io.Serial; +import java.io.Serializable; + +@Builder(toBuilder = true) +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Entity +@Table(name = "wc_file_extension") +@JsonIgnoreProperties({ "turWCSource" }) +public class TurWCFileExtension implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + @Id + @TurUuid + @Column(name = "id", nullable = false) + private String id; + + private String extension; + + // bi-directional many-to-one association to TurWCSource + @ManyToOne + @JoinColumn(name = "ws_source_id", nullable = false) + private TurWCSource turWCSource; + + public TurWCFileExtension(String extension, TurWCSource turWCSource) { + this.extension = extension; + this.turWCSource = turWCSource; + } +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCNotAllowUrl.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCNotAllowUrl.java new file mode 100644 index 00000000000..d68188ecab8 --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCNotAllowUrl.java @@ -0,0 +1,37 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import jakarta.persistence.Entity; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; + +import java.io.Serial; +import java.io.Serializable; + +@SuperBuilder +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Entity +@JsonIgnoreProperties({ "turWCSource" }) +public class TurWCNotAllowUrl extends TurWCUrl implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + // bi-directional many-to-one association to TurWCSource + @ManyToOne + @JoinColumn(name = "wc_source_id", nullable = false) + private TurWCSource turWCSource; + + public TurWCNotAllowUrl(String url, TurWCSource turWCSource) { + this.url = url; + this.turWCSource = turWCSource; + } +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCSource.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCSource.java new file mode 100644 index 00000000000..bba0243fd9c --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCSource.java @@ -0,0 +1,127 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.model; + +import com.viglet.turing.spring.jpa.TurUuid; +import jakarta.persistence.*; +import lombok.*; +import org.hibernate.annotations.Cascade; +import org.hibernate.annotations.OnDelete; +import org.hibernate.annotations.OnDeleteAction; + +import java.io.Serial; +import java.io.Serializable; +import java.util.Collection; +import java.util.HashSet; +import java.util.Locale; + +@Builder(toBuilder = true) +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Entity +@Table(name = "wc_source") +public class TurWCSource implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + public static final String TUR_WC_SOURCE = "turWCSource"; + public static final String SOURCE_ID = "source_id"; + public static final String WC_SN_SITE = "wc_sn_site"; + public static final String SN_SITE = "sn_site"; + + @Id + @TurUuid + @Column(name = "id", nullable = false) + private String id; + @Column + private String title; + @Column + private String description; + @Column + private Locale locale; + @Column + private String localeClass; + @Column + private String url; + @Column + private String username; + @Column + private String password; + + @Builder.Default + @ElementCollection(targetClass = String.class, fetch = FetchType.EAGER) + @CollectionTable(name = WC_SN_SITE, joinColumns = @JoinColumn(name = SOURCE_ID)) + @Column(name = SN_SITE, nullable = false) + private Collection turSNSites = new HashSet<>(); + + @Builder.Default + @OneToMany(mappedBy = TUR_WC_SOURCE, orphanRemoval = true, fetch = FetchType.LAZY) + @Cascade({org.hibernate.annotations.CascadeType.ALL, org.hibernate.annotations.CascadeType.DELETE_ORPHAN}) + @OnDelete(action = OnDeleteAction.CASCADE) + private Collection startingPoints = new HashSet<>(); + + @Builder.Default + @OneToMany(mappedBy = TUR_WC_SOURCE, orphanRemoval = true, fetch = FetchType.LAZY) + @Cascade({org.hibernate.annotations.CascadeType.ALL, org.hibernate.annotations.CascadeType.DELETE_ORPHAN}) + @OnDelete(action = OnDeleteAction.CASCADE) + private Collection allowUrls = new HashSet<>(); + + @Builder.Default + @OneToMany(mappedBy = TUR_WC_SOURCE, orphanRemoval = true, fetch = FetchType.LAZY) + @Cascade({org.hibernate.annotations.CascadeType.ALL, org.hibernate.annotations.CascadeType.DELETE_ORPHAN}) + @OnDelete(action = OnDeleteAction.CASCADE) + private Collection notAllowUrls = new HashSet<>(); + + @Builder.Default + @OneToMany(mappedBy = TUR_WC_SOURCE, orphanRemoval = true, fetch = FetchType.LAZY) + @Cascade({org.hibernate.annotations.CascadeType.ALL, org.hibernate.annotations.CascadeType.DELETE_ORPHAN}) + @OnDelete(action = OnDeleteAction.CASCADE) + private Collection notAllowExtensions = new HashSet<>(); + + @Builder.Default + @OneToMany(mappedBy = TUR_WC_SOURCE, orphanRemoval = true, fetch = FetchType.LAZY) + @Cascade({org.hibernate.annotations.CascadeType.ALL, org.hibernate.annotations.CascadeType.DELETE_ORPHAN}) + @OnDelete(action = OnDeleteAction.CASCADE) + private Collection attributeMappings = new HashSet<>(); + + public void setStartingPoints(Collection startingPoints) { + this.startingPoints.clear(); + if (startingPoints != null) { + this.startingPoints.addAll(startingPoints); + } + } + + + public void setAllowUrls(Collection allowUrls) { + this.allowUrls.clear(); + if (allowUrls != null) { + this.allowUrls.addAll(allowUrls); + } + } + + public void setNotAllowUrls(Collection notAllowUrls) { + this.notAllowUrls.clear(); + if (notAllowUrls != null) { + this.notAllowUrls.addAll(notAllowUrls); + } + } + public void setNotAllowExtensions(Collection notAllowExtensions) { + this.notAllowExtensions.clear(); + if (notAllowExtensions != null) { + this.notAllowExtensions.addAll(notAllowExtensions); + } + } + + public void setAttributeMappings(Collection attributeMappings) { + this.attributeMappings.clear(); + if (attributeMappings != null) { + this.attributeMappings.addAll(attributeMappings); + } + } + public void setTurSNSites(Collection turSNSites) { + this.turSNSites.clear(); + if (turSNSites != null) { + this.turSNSites.addAll(turSNSites); + } + } +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCStartingPoint.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCStartingPoint.java new file mode 100644 index 00000000000..6ef5b01e417 --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCStartingPoint.java @@ -0,0 +1,37 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import jakarta.persistence.Entity; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; + +import java.io.Serial; +import java.io.Serializable; + +@SuperBuilder +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Entity +@JsonIgnoreProperties({ "turWCSource" }) +public class TurWCStartingPoint extends TurWCUrl implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + // bi-directional many-to-one association to TurWCSource + @ManyToOne + @JoinColumn(name = "wc_source_id", nullable = false) + private TurWCSource turWCSource; + + public TurWCStartingPoint(String url, TurWCSource turWCSource) { + this.url = url; + this.turWCSource = turWCSource; + } +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCUrl.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCUrl.java new file mode 100644 index 00000000000..2723be9838e --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/model/TurWCUrl.java @@ -0,0 +1,33 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.model; + +import com.viglet.turing.spring.jpa.TurUuid; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; + +import java.io.Serial; +import java.io.Serializable; + +@SuperBuilder +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Entity +@Table(name = "wc_url") +public class TurWCUrl implements Serializable { + @Serial + private static final long serialVersionUID = 1L; + + @Id + @TurUuid + @Column(name = "id", nullable = false) + protected String id; + protected String url; +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCAllowUrlRepository.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCAllowUrlRepository.java new file mode 100644 index 00000000000..e003a71f77b --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCAllowUrlRepository.java @@ -0,0 +1,12 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.repository; + +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCAllowUrl; +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCSource; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface TurWCAllowUrlRepository extends JpaRepository { + Optional> findByTurWCSource(TurWCSource turWCSource); +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCAttributeMappingRepository.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCAttributeMappingRepository.java new file mode 100644 index 00000000000..fd1219d28d6 --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCAttributeMappingRepository.java @@ -0,0 +1,12 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.repository; + +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCAttributeMapping; +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCSource; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface TurWCAttributeMappingRepository extends JpaRepository { + Optional> findByTurWCSource(TurWCSource turWCSource); +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCFileExtensionRepository.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCFileExtensionRepository.java new file mode 100644 index 00000000000..b66ed0821e7 --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCFileExtensionRepository.java @@ -0,0 +1,12 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.repository; + +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCFileExtension; +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCSource; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface TurWCFileExtensionRepository extends JpaRepository { + Optional> findByTurWCSource(TurWCSource turWCSource); +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCNotAllowUrlRepository.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCNotAllowUrlRepository.java new file mode 100644 index 00000000000..a2d990756a4 --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCNotAllowUrlRepository.java @@ -0,0 +1,12 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.repository; + +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCNotAllowUrl; +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCSource; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface TurWCNotAllowUrlRepository extends JpaRepository { + Optional> findByTurWCSource(TurWCSource turWCSource); +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCSourceRepository.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCSourceRepository.java new file mode 100644 index 00000000000..eba92eac062 --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCSourceRepository.java @@ -0,0 +1,7 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.repository; + +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCSource; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface TurWCSourceRepository extends JpaRepository { +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCStartingPointRepository.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCStartingPointRepository.java new file mode 100644 index 00000000000..b5ac289166f --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCStartingPointRepository.java @@ -0,0 +1,12 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.repository; + +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCSource; +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCStartingPoint; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface TurWCStartingPointRepository extends JpaRepository { + Optional> findByTurWCSource(TurWCSource turWCSource); +} diff --git a/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCUrlRepository.java b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCUrlRepository.java new file mode 100644 index 00000000000..8cab798fd57 --- /dev/null +++ b/turing-web-crawler/wc-plugin/src/main/java/com/viglet/turing/connector/plugin/webcrawler/persistence/repository/TurWCUrlRepository.java @@ -0,0 +1,7 @@ +package com.viglet.turing.connector.plugin.webcrawler.persistence.repository; + +import com.viglet.turing.connector.plugin.webcrawler.persistence.model.TurWCUrl; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface TurWCUrlRepository extends JpaRepository { +} diff --git a/turing-web-crawler/wc-sample/pom.xml b/turing-web-crawler/wc-sample/pom.xml index 5f089fdfe6a..112dbcf68ab 100644 --- a/turing-web-crawler/wc-sample/pom.xml +++ b/turing-web-crawler/wc-sample/pom.xml @@ -24,7 +24,7 @@ org.jsoup jsoup - 1.18.1 + 1.18.3 diff --git a/turing-wem/pom.xml b/turing-wem/pom.xml index 954eef21f29..b9331fd2a3a 100644 --- a/turing-wem/pom.xml +++ b/turing-wem/pom.xml @@ -38,7 +38,7 @@ com.fasterxml.jackson.core jackson-databind - 2.18.1 + 2.18.2 org.apache.httpcomponents diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TurAttrDef.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TurAttrDef.java index 2248bca378b..66e6b149c0b 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TurAttrDef.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TurAttrDef.java @@ -17,33 +17,27 @@ */ package com.viglet.turing.connector.wem.beans; +import lombok.Getter; +import lombok.Setter; + import java.util.List; +@Setter public class TurAttrDef { - private String tagName; + @Getter + private String tagName; private TurMultiValue multiValue; public TurAttrDef (String tagName, TurMultiValue multiValue) { this.tagName = tagName; this.multiValue = multiValue; } - public String getTagName() { - return tagName; - } - public void setTagName(String tagName) { - this.tagName = tagName; - } - - public List getMultiValue() { + public List getMultiValue() { return multiValue; } - public void setMultiValue(TurMultiValue multiValue) { - this.multiValue = multiValue; - } - - public String toString() { + public String toString() { return String.format("tagName: %s, multiValue: %s", tagName, multiValue); } } diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TurAttrDefContext.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TurAttrDefContext.java index 0d53fc38e82..18f514f24d1 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TurAttrDefContext.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TurAttrDefContext.java @@ -25,20 +25,27 @@ import com.vignette.as.client.exception.ApplicationException; import com.vignette.as.client.javabean.ContentInstance; import com.vignette.logging.context.ContextLogger; +import lombok.Getter; +import lombok.Setter; +@Setter public class TurAttrDefContext { private static final ContextLogger logger = ContextLogger.getLogger(MethodHandles.lookup().lookupClass()); - private ContentInstance contentInstance; - private TuringTag turingTag; - private IHandlerConfiguration iHandlerConfiguration; - private MappingDefinitions mappingDefinitions; + @Getter + private ContentInstance contentInstance; + @Getter + private TuringTag turingTag; + @Getter + private IHandlerConfiguration iHandlerConfiguration; + @Getter + private MappingDefinitions mappingDefinitions; private AttributeData attributeData; public TurAttrDefContext(TurAttrDefContext turAttrDefContext) { this.setAttributeData(turAttrDefContext.getAttributeData()); this.setContentInstance(turAttrDefContext.getContentInstance()); - this.setiHandlerConfiguration(turAttrDefContext.getiHandlerConfiguration()); + this.setIHandlerConfiguration(turAttrDefContext.getIHandlerConfiguration()); this.setMappingDefinitions(turAttrDefContext.getMappingDefinitions()); this.setTuringTag(turAttrDefContext.getTuringTag()); } @@ -46,44 +53,12 @@ public TurAttrDefContext(TurAttrDefContext turAttrDefContext) { public TurAttrDefContext(ContentInstance contentInstance, TuringTag turingTag, IHandlerConfiguration iHandlerConfiguration, MappingDefinitions mappingDefinitions) { this.setContentInstance(contentInstance); - this.setiHandlerConfiguration(iHandlerConfiguration); + this.setIHandlerConfiguration(iHandlerConfiguration); this.setMappingDefinitions(mappingDefinitions); this.setTuringTag(turingTag); } - public ContentInstance getContentInstance() { - return contentInstance; - } - - public void setContentInstance(ContentInstance contentInstance) { - this.contentInstance = contentInstance; - } - - public TuringTag getTuringTag() { - return turingTag; - } - - public void setTuringTag(TuringTag turingTag) { - this.turingTag = turingTag; - } - - public IHandlerConfiguration getiHandlerConfiguration() { - return iHandlerConfiguration; - } - - public void setiHandlerConfiguration(IHandlerConfiguration iHandlerConfiguration) { - this.iHandlerConfiguration = iHandlerConfiguration; - } - - public MappingDefinitions getMappingDefinitions() { - return mappingDefinitions; - } - - public void setMappingDefinitions(MappingDefinitions mappingDefinitions) { - this.mappingDefinitions = mappingDefinitions; - } - - public AttributeData getAttributeData() { + public AttributeData getAttributeData() { if (attributeData != null) return attributeData; @@ -96,7 +71,4 @@ public AttributeData getAttributeData() { return null; } - public void setAttributeData(AttributeData attributeData) { - this.attributeData = attributeData; - } } diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TurMultiValue.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TurMultiValue.java index 4ab36cc654f..03fbf32cd45 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TurMultiValue.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TurMultiValue.java @@ -33,7 +33,7 @@ public static TurMultiValue singleItem(String text) { public static TurMultiValue fromList(List list) { TurMultiValue turMultiValue = new TurMultiValue(); - list.forEach(turMultiValue::add); + turMultiValue.addAll(list); return turMultiValue; } } diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TuringTag.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TuringTag.java index 293920fd9bf..05c9ddd8daf 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TuringTag.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TuringTag.java @@ -17,17 +17,27 @@ */ package com.viglet.turing.connector.wem.beans; +import lombok.Getter; +import lombok.Setter; + import java.util.List; +@Setter public class TuringTag { - private String tagName; - private List srcAttributeRelation; - private String srcAttributeType; - private String srcXmlName; - private String srcClassName; + @Getter + private String tagName; + @Getter + private List srcAttributeRelation; + @Getter + private String srcAttributeType; + @Getter + private String srcXmlName; + @Getter + private String srcClassName; private boolean srcMandatory; private boolean srcUniqueValues; - private String textValue; + @Getter + private String textValue; @Override public String toString() { @@ -41,63 +51,11 @@ public TuringTag() { super(); } - public String getSrcAttributeType() { - return srcAttributeType; - } - - public List getSrcAttributeRelation() { - return srcAttributeRelation; - } - - public String getSrcClassName() { - return srcClassName; - } - - public boolean getSrcMandatory() { + public boolean getSrcMandatory() { return srcMandatory; } - public String getTagName() { - return tagName; - } - - public String getTextValue() { - return textValue; - } - - public void setSrcAttributeType(String srcAttributeType) { - this.srcAttributeType = srcAttributeType; - } - - public void setSrcAttributeRelation(List srcAttributeRelation) { - this.srcAttributeRelation = srcAttributeRelation; - } - - public void setSrcClassName(String srcClassName) { - this.srcClassName = srcClassName; - } - - public void setSrcMandatory(boolean srcMandatory) { - this.srcMandatory = srcMandatory; - } - - public void setTagName(String tagName) { - this.tagName = tagName; - } - - public String getSrcXmlName() { - return srcXmlName; - } - - public void setSrcXmlName(String srcXmlName) { - this.srcXmlName = srcXmlName; - } - - public void setTextValue(String textValue) { - this.textValue = textValue; - } - - @Override + @Override public boolean equals(Object obj) { if (obj instanceof TuringTag) { return this.getTagName().equals(((TuringTag) obj).getTagName()); @@ -115,7 +73,4 @@ public boolean isSrcUniqueValues() { return srcUniqueValues; } - public void setSrcUniqueValues(boolean srcUniqueValues) { - this.srcUniqueValues = srcUniqueValues; - } } diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TuringTagMap.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TuringTagMap.java index 7462c9dd247..9e3ff14f43a 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TuringTagMap.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/beans/TuringTagMap.java @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.HashMap; -//HashMap turingTags> public class TuringTagMap extends HashMap>{ private static final long serialVersionUID = 1L; diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/attribute/TurWEMAttrClass.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/attribute/TurWEMAttrClass.java index fa107bd3cc5..c784a9bc671 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/attribute/TurWEMAttrClass.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/attribute/TurWEMAttrClass.java @@ -46,7 +46,7 @@ public static List attributeByClass(TurAttrDefContext turAttrDefCont TuringTag turingTag = turAttrDefContext.getTuringTag(); ContentInstance ci = turAttrDefContext.getContentInstance(); - IHandlerConfiguration config = turAttrDefContext.getiHandlerConfiguration(); + IHandlerConfiguration config = turAttrDefContext.getIHandlerConfiguration(); List attributesDefs = new ArrayList<>(); if (turingTag.getSrcClassName() != null) { diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/attribute/TurWEMAttrXML.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/attribute/TurWEMAttrXML.java index acb6f69a12d..d44cbe7422f 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/attribute/TurWEMAttrXML.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/attribute/TurWEMAttrXML.java @@ -67,7 +67,7 @@ public static List attributeXMLUpdate(TurAttrDefContext turAttrDefCo log.debug(String.format("%s = %s", turingTag.getTagName(), attributeData.getValue().toString())); if (attributeData != null && attributeData.getValue().toString() != null - && attributeData.getValue().toString().trim().length() > 0) + && !attributeData.getValue().toString().trim().isEmpty()) return TurWEMAttrWidget.attributeByWidget(turAttrDefContext, attributeData); return new ArrayList<>(); @@ -105,7 +105,7 @@ private static void addRelationAttributes(TurAttrDefContext turAttrDefContext, S AttributeData attributeData = attributedObject.getAttribute(attributeName); if (log.isDebugEnabled()) log.debug(String.format("Attribute: %s, Value: %s", attributeName, attributeValue)); - if (attributeValue != null && attributeValue.trim().length() > 0) + if (attributeValue != null && !attributeValue.trim().isEmpty()) attributesDefs.addAll(attributeXMLUpdate(turAttrDefContext, attributeData)); } } @@ -117,7 +117,7 @@ private static List addAttributeWithoutRelator(TurAttrDefContext tur ContentInstance ci = turAttrDefContext.getContentInstance(); String attributeName = turAttrDefContext.getTuringTag().getSrcXmlName(); if (ci.getAttributeValue(attributeName) != null - && ci.getAttributeValue(attributeName).toString().trim().length() > 0) { + && !ci.getAttributeValue(attributeName).toString().trim().isEmpty()) { AttributeData attributeData = ci.getAttribute(attributeName); return attributeXMLUpdate(turAttrDefContext, attributeData); } else if (turingTag.getSrcClassName() != null) { diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/relator/TurWEMRelator.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/relator/TurWEMRelator.java index 0f64211d47f..c2cb093ee44 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/relator/TurWEMRelator.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/relator/TurWEMRelator.java @@ -65,7 +65,7 @@ private static AttributedObject[] detectAttributesFromRelator(List relat List nestedRelationChild = new ArrayList<>(); for (AttributedObject[] attributesFromRelation : currentRelation) { - for (AttributedObject attributeFromRelation : Arrays.asList(attributesFromRelation)) { + for (AttributedObject attributeFromRelation : attributesFromRelation) { try { AttributedObject[] childRelation = attributeFromRelation .getRelations(relationTag.get(nextPosition)); diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/update/TurWEMUpdateFileWidget.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/update/TurWEMUpdateFileWidget.java index 314e3da2943..5f476aa7402 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/update/TurWEMUpdateFileWidget.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/broker/update/TurWEMUpdateFileWidget.java @@ -44,7 +44,7 @@ public static List attributeFileWidgetUpdate(TurAttrDefContext turAt TuringTag turingTag = turAttrDefContext.getTuringTag(); ContentInstance ci = turAttrDefContext.getContentInstance(); - IHandlerConfiguration config = turAttrDefContext.getiHandlerConfiguration(); + IHandlerConfiguration config = turAttrDefContext.getIHandlerConfiguration(); if (log.isDebugEnabled()) { log.debug("TurWEMUpdateFileWidget started"); diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/config/GenericResourceHandlerConfiguration.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/config/GenericResourceHandlerConfiguration.java index 6fa78991c8d..86e6902ed4a 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/config/GenericResourceHandlerConfiguration.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/config/GenericResourceHandlerConfiguration.java @@ -23,6 +23,7 @@ import com.vignette.logging.context.ContextLogger; import com.vignette.util.CustomerMsg; import com.vignette.util.MsgObject; +import lombok.Setter; import java.io.IOException; import java.io.StringReader; @@ -49,6 +50,7 @@ public class GenericResourceHandlerConfiguration implements IHandlerConfiguratio private static final String DEFAULT_SN_LOCALE = "en_US"; private static final String DEFAULT_DPS_CONTEXT = "sites"; + @Setter private String turingURL; private String snSite; private String snLocale; @@ -57,7 +59,9 @@ public class GenericResourceHandlerConfiguration implements IHandlerConfiguratio private String cdaURLPrefix; private String sitesAssociationPriority; private String fileSourcePath; + @Setter private String login; + @Setter private String password; private String providerName; private static final ContextLogger log = ContextLogger.getLogger(MethodHandles.lookup().lookupClass()); @@ -72,10 +76,6 @@ public String getTuringURL() { return turingURL; } - public void setTuringURL(String turingURL) { - this.turingURL = turingURL; - } - @Override public String getMappingsXML() { return mappingsXML; @@ -286,19 +286,11 @@ public String getLogin() { return login; } - public void setLogin(String login) { - this.login = login; - } - @Override public String getPassword() { return password; } - public void setPassword(String password) { - this.password = password; - } - @Override public String getProviderName() { return providerName; diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/config/TurSNSiteConfig.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/config/TurSNSiteConfig.java index 183d38a5a45..04a77f98030 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/config/TurSNSiteConfig.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/config/TurSNSiteConfig.java @@ -17,6 +17,11 @@ */ package com.viglet.turing.connector.wem.config; +import lombok.Getter; +import lombok.Setter; + +@Setter +@Getter public class TurSNSiteConfig { private String name; @@ -28,20 +33,4 @@ public TurSNSiteConfig(String name, String locale) { this.locale = locale; } - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getLocale() { - return locale; - } - - public void setLocale(String locale) { - this.locale = locale; - } - } diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/ext/TurSpotlightExtraFields.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/ext/TurSpotlightExtraFields.java index 804ebc3a048..36a069038ff 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/ext/TurSpotlightExtraFields.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/ext/TurSpotlightExtraFields.java @@ -29,11 +29,12 @@ import com.vignette.as.client.javabean.ContentInstance; import com.vignette.as.client.javabean.ManagedObject; import com.vignette.logging.context.ContextLogger; +import lombok.Getter; +import lombok.Setter; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; public class TurSpotlightExtraFields implements ExtAttributeInterface { @@ -51,7 +52,7 @@ public TurMultiValue consume(TuringTag tag, ContentInstance ci, AttributeData at Object title = attributedObject.getAttributeValue("TITLE-TUR-SPOTLIGHT-CONTENT"); Object content = attributedObject.getAttributeValue("CONTENT-TUR-SPOTLIGHT-CONTENT"); Object link = attributedObject.getAttributeValue("LINK-TUR-SPOTLIGHT-CONTENT"); - String contentText = ""; + String contentText; if (((String) content).length() == 40 ) { ContentInstance contentInstance = (ContentInstance) ManagedObject .findByContentManagementId(new ManagedObjectVCMRef(content.toString())); @@ -78,27 +79,16 @@ public TurMultiValue consume(TuringTag tag, ContentInstance ci, AttributeData at return new TurMultiValue(); } + @Setter + @Getter static class TurCTDAttributes { private String id; private String title; - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } } + @Setter + @Getter static class TurSpotlightContent { private int position; private String title; @@ -113,37 +103,5 @@ public TurSpotlightContent(int position, String title, String content, String li this.link = link; } - public int getPosition() { - return position; - } - - public void setPosition(int position) { - this.position = position; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getContent() { - return content; - } - - public void setContent(String content) { - this.content = content; - } - - public String getLink() { - return link; - } - - public void setLink(String link) { - this.link = link; - } - } } diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/listener/DeploymentHandler.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/listener/DeploymentHandler.java index f65311f57d1..fd0635cb35c 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/listener/DeploymentHandler.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/listener/DeploymentHandler.java @@ -29,7 +29,7 @@ public class DeploymentHandler { private static final ContextLogger log = ContextLogger.getLogger(MethodHandles.lookup().lookupClass()); - private IHandlerConfiguration config; + private final IHandlerConfiguration config; public DeploymentHandler(IHandlerConfiguration config) { this.config = config; diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/listener/PrePersistenceHandler.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/listener/PrePersistenceHandler.java index 2a47abf926f..c934ac74e34 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/listener/PrePersistenceHandler.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/listener/PrePersistenceHandler.java @@ -28,7 +28,7 @@ public class PrePersistenceHandler { private static final ContextLogger log = ContextLogger.getLogger(MethodHandles.lookup().lookupClass()); - private IHandlerConfiguration config; + private final IHandlerConfiguration config; public PrePersistenceHandler(IHandlerConfiguration config) { this.config = config; diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/mappers/CTDMappings.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/mappers/CTDMappings.java index cc6ff4a6a22..2bf5f56c842 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/mappers/CTDMappings.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/mappers/CTDMappings.java @@ -27,9 +27,11 @@ import com.viglet.turing.connector.wem.beans.TuringTagMap; import com.viglet.turing.connector.wem.util.TuringUtils; import com.vignette.logging.context.ContextLogger; +import lombok.Getter; +@Getter public class CTDMappings { - private TuringTagMap turingTagMap; + private final TuringTagMap turingTagMap; private String classValidToIndex = null; private static final ContextLogger log = ContextLogger.getLogger(MethodHandles.lookup().lookupClass()); @@ -71,15 +73,7 @@ public CTDMappings(TuringTagMap turingTagMap) { this.turingTagMap = turingTagMap; } - public TuringTagMap getTuringTagMap() { - return turingTagMap; - } - - public String getClassValidToIndex() { - return classValidToIndex; - } - - public void setClassValidToIndex(String classValidToIndex) { + public void setClassValidToIndex(String classValidToIndex) { this.classValidToIndex = classValidToIndex; } diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/mappers/MappingDefinitions.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/mappers/MappingDefinitions.java index eaaf4704f58..74458b5875a 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/mappers/MappingDefinitions.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/mappers/MappingDefinitions.java @@ -25,21 +25,19 @@ import com.vignette.as.client.javabean.ContentInstance; import com.vignette.as.client.javabean.ObjectType; import com.vignette.logging.context.ContextLogger; +import lombok.Setter; public class MappingDefinitions { private TurCTDMappingMap turCTDMappingMap; - private String mappingsXML; + @Setter + private String mappingsXML; private static final ContextLogger logger = ContextLogger.getLogger(MethodHandles.lookup().lookupClass()); public String getMappingsXML() { return mappingsXML; } - public void setMappingsXML(String mappingsXML) { - this.mappingsXML = mappingsXML; - } - - public MappingDefinitions() { + public MappingDefinitions() { } public MappingDefinitions(String mappingsXML, TurCTDMappingMap turCTDMappingMap) { diff --git a/turing-wem/src/main/java/com/viglet/turing/connector/wem/util/TuringUtils.java b/turing-wem/src/main/java/com/viglet/turing/connector/wem/util/TuringUtils.java index d1390db0fc5..a4cb855596d 100644 --- a/turing-wem/src/main/java/com/viglet/turing/connector/wem/util/TuringUtils.java +++ b/turing-wem/src/main/java/com/viglet/turing/connector/wem/util/TuringUtils.java @@ -109,11 +109,9 @@ public static void sendToTuringAsZipFile(ByteArrayOutputStream byteArrayOutputSt try (CloseableHttpResponse response = client.execute(httpPost)) { if (log.isDebugEnabled()) { - log.debug(String.format("Viglet Turing Index Request URI: %s", httpPost.getURI())); - log.debug(String.format("Viglet Turing indexer response HTTP result is: %s, for request uri: %s", - response.getStatusLine().getStatusCode(), httpPost.getURI())); - log.debug(String.format("Viglet Turing indexer response HTTP result is: %s", - httpPost.getEntity().toString())); + log.debug("Viglet Turing Index Request URI: {}", httpPost.getURI()); + log.debug("Viglet Turing indexer response HTTP result is: {}, for request uri: {}", response.getStatusLine().getStatusCode(), httpPost.getURI()); + log.debug("Viglet Turing indexer response HTTP result is: {}", httpPost.getEntity().toString()); } } } catch (IOException e) { @@ -142,28 +140,16 @@ public static void sendToTuring(TurSNJobItems turSNJobItems, IHandlerConfigurati // encode ByteBuffer outputBuffer = customCharset.encode(data); - byte[] outputData = new String(outputBuffer.array()).getBytes(StandardCharsets.UTF_8); - String jsonUTF8 = new String(outputData); - HttpPost httpPost = new HttpPost( - String.format("%s/api/sn/%s/import", config.getTuringURL(), turSNSiteConfig.getName())); - - StringEntity entity = new StringEntity(jsonUTF8, StandardCharsets.UTF_8); - httpPost.setEntity(entity); - httpPost.setHeader("Accept", "application/json"); - httpPost.setHeader("Content-type", "application/json"); - httpPost.setHeader("Accept-Encoding", StandardCharsets.UTF_8.name()); + HttpPost httpPost = getHttpPost(config, turSNSiteConfig, outputBuffer); basicAuth(config, httpPost); try (CloseableHttpResponse response = client.execute(httpPost)) { if (log.isDebugEnabled()) { - log.debug(String.format("Viglet Turing Index Request URI: %s", httpPost.getURI())); - log.debug(String.format("JSON: %s", jsonResult)); - log.debug( - String.format("Viglet Turing indexer response HTTP result is: %s, for request uri: %s", - response.getStatusLine().getStatusCode(), httpPost.getURI())); - log.debug(String.format("Viglet Turing indexer response HTTP result is: %s", - httpPost.getEntity().toString())); + log.debug("Viglet Turing Index Request URI: {}", httpPost.getURI()); + log.debug("JSON: {}", jsonResult); + log.debug("Viglet Turing indexer response HTTP result is: {}, for request uri: {}", response.getStatusLine().getStatusCode(), httpPost.getURI()); + log.debug("Viglet Turing indexer response HTTP result is: {}", httpPost.getEntity().toString()); } turSNJobItems.getTuringDocuments().clear(); } @@ -173,6 +159,20 @@ public static void sendToTuring(TurSNJobItems turSNJobItems, IHandlerConfigurati } } + private static HttpPost getHttpPost(IHandlerConfiguration config, TurSNSiteConfig turSNSiteConfig, ByteBuffer outputBuffer) { + byte[] outputData = new String(outputBuffer.array()).getBytes(StandardCharsets.UTF_8); + String jsonUTF8 = new String(outputData); + HttpPost httpPost = new HttpPost( + String.format("%s/api/sn/%s/import", config.getTuringURL(), turSNSiteConfig.getName())); + + StringEntity entity = new StringEntity(jsonUTF8, StandardCharsets.UTF_8); + httpPost.setEntity(entity); + httpPost.setHeader("Accept", "application/json"); + httpPost.setHeader("Content-type", "application/json"); + httpPost.setHeader("Accept-Encoding", StandardCharsets.UTF_8.name()); + return httpPost; + } + public static Channel getChosenChannel(ChannelRef[] channelRefs, List siteNames, IHandlerConfiguration config) throws ApplicationException, ValidationException, RemoteException { String siteName = null; @@ -397,7 +397,7 @@ public static String getSiteUrl(ManagedObject mo, IHandlerConfiguration config) return createSiteURL(mo, config); } else { if (log.isDebugEnabled()) { - log.debug("ETLTuringTranslator Content without channel:" + mo.getName()); + log.debug("ETLTuringTranslator Content without channel:{}", mo.getName()); } return null; }