From 23093c3ca0bf38e81b297aa9e0113257dfb90c69 Mon Sep 17 00:00:00 2001 From: jaumeortola Date: Thu, 11 Apr 2024 16:03:22 +0200 Subject: [PATCH 1/3] disable some French rules if model is available --- .../java/org/languagetool/JLanguageTool.java | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/languagetool-core/src/main/java/org/languagetool/JLanguageTool.java b/languagetool-core/src/main/java/org/languagetool/JLanguageTool.java index 7a1e5df367cb..321f452d1a07 100644 --- a/languagetool-core/src/main/java/org/languagetool/JLanguageTool.java +++ b/languagetool-core/src/main/java/org/languagetool/JLanguageTool.java @@ -1008,11 +1008,12 @@ protected CheckResults checkInternal(AnnotatedText annotatedText, ParagraphHandl paraMode, annotatedText, listener, mode, level, remoteRulesThreadPool == null, toneTags); long textCheckEnd = System.nanoTime(); + Map remoteRulesResults = null; try { - TelemetryProvider.INSTANCE.createSpan("fetch-remote-rules", + remoteRulesResults = TelemetryProvider.INSTANCE.createSpan("fetch-remote-rules", Attributes.builder().put("check.remote_rules.count", remoteRules.size()).build(), () -> fetchRemoteRuleResults(deadlineStartNanos, mode, level, analyzedSentences, remoteMatches, remoteRuleTasks, remoteRules, requestSize, - cachedResults, matchOffset, annotatedText, textSessionID, toneTags)); + cachedResults, matchOffset, annotatedText, textSessionID, toneTags)); } catch (Exception e) { throw new RuntimeException(e); } @@ -1034,7 +1035,7 @@ protected CheckResults checkInternal(AnnotatedText annotatedText, ParagraphHandl return res; } - ruleMatches = filterMatches(annotatedText, rules, ruleMatches, level, toneTags); + ruleMatches = filterMatches(annotatedText, rules, ruleMatches, level, toneTags, remoteRulesResults); // decide if this should be done right after performCheck, before waiting for remote rule results // better for latency, remote rules probably don't need resorting @@ -1058,12 +1059,16 @@ protected CheckResults checkInternal(AnnotatedText annotatedText, ParagraphHandl return new CheckResults(ruleMatches, res.getIgnoredRanges(), res.getExtendedSentenceRanges()); } - private List filterMatches(AnnotatedText annotatedText, RuleSet rules, List ruleMatches, Level level, Set toneTags) { + private List filterMatches(AnnotatedText annotatedText, RuleSet rules, List ruleMatches, Level level, + Set toneTags, Map remoteRulesResults) { // rules can create matches with rule IDs different from the original rule (see e.g. RemoteRules) // so while we can't avoid execution of these rules, we still want disabling them to work // so do another pass with ignoreRule here ruleMatches = ruleMatches.stream().filter(match -> !ignoreRule(match.getRule())).collect(Collectors.toList()); + ruleMatches = ruleMatches.stream().filter(match -> isRuleActiveForLanguageWithModel( + match.getRule(), language, remoteRulesResults)).collect(Collectors.toList()); + ruleMatches = ruleMatches.stream().filter(match -> isRuleActiveForLevelAndToneTags( match.getRule(), level, toneTags)).collect(Collectors.toList()); @@ -1078,6 +1083,19 @@ private List filterMatches(AnnotatedText annotatedText, RuleSet rules return applyCustomFilters(ruleMatches, annotatedText); } + private boolean isRuleActiveForLanguageWithModel(Rule rule, Language language, Map remoteRulesResults) { + if (language.getShortCode().equals("fr")) { + List disableFrenchRules = Arrays.asList("OU"); + RemoteRuleResult remoteRulesResult = remoteRulesResults.get("AI_FR_GGEC"); + if (remoteRulesResult != null) { + if (remoteRulesResult.isSuccess()) { + return !disableFrenchRules.contains(rule.getId()); + } + } + } + return false; + } + private final Map ruleSetCache = new ConcurrentHashMap<>(); static boolean isRuleActiveForLevelAndToneTags(Rule rule, Level level, Set toneTags) { @@ -1122,13 +1140,14 @@ private RuleSet getActiveRulesForLevelAndToneTags(Level level, Set tone }); } - protected void fetchRemoteRuleResults(long deadlineStartNanos, Mode mode, Level level, List analyzedSentences, List remoteMatches, + protected Map fetchRemoteRuleResults(long deadlineStartNanos, Mode mode, Level level, List analyzedSentences, List remoteMatches, List> remoteRuleTasks, List remoteRules, List requestSize, Map> cachedResults, Map matchOffset, AnnotatedText annotatedText, Long textSessionID, Set toneTags) { + Map remoteRuleResults = new HashMap<>(); if (remoteRuleTasks != null && !remoteRuleTasks.isEmpty()) { int timeout = IntStream.range(0, requestSize.size()).map(i -> (int) remoteRules.get(i).getTimeout(requestSize.get(i)) @@ -1153,13 +1172,17 @@ protected void fetchRemoteRuleResults(long deadlineStartNanos, Mode mode, Level RemoteRuleMetrics.request(ruleKey, deadlineStartNanos, chars, RemoteRuleMetrics.RequestResult.DOWN); continue; } + RemoteRuleResult remoteRuleResult = null; try { //logger.info("Fetching results for remote rule for {} chars", chars); - RemoteRuleMetrics.inCircuitBreaker(deadlineStartNanos, rule.circuitBreaker(), ruleKey, chars, () -> + remoteRuleResult = RemoteRuleMetrics.inCircuitBreaker(deadlineStartNanos, rule.circuitBreaker(), ruleKey, chars, () -> fetchResults(deadlineStartNanos, mode, level, analyzedSentences, remoteMatches, matchOffset, annotatedText, textSessionID, chars, deadlineEndNanos, task, rule, ruleKey, toneTags)); } catch (InterruptedException e) { break; } + if (remoteRuleResult != null) { + remoteRuleResults.put(ruleKey, remoteRuleResult); + } } for (Integer cachedSentenceIndex : cachedResults.keySet()) { @@ -1180,6 +1203,7 @@ protected void fetchRemoteRuleResults(long deadlineStartNanos, Mode mode, Level // cancel any remaining tasks (e.g. after interrupt because request timed out) remoteRuleTasks.stream().filter(Objects::nonNull).forEach(t -> t.cancel(true)); } + return remoteRuleResults; } private RemoteRuleResult fetchResults(long deadlineStartNanos, Mode mode, Level level, List analyzedSentences, List remoteMatches, Map matchOffset, AnnotatedText annotatedText, Long textSessionID, long chars, long deadlineEndNanos, FutureTask task, RemoteRule rule, String ruleKey, Set toneTags) throws InterruptedException, ExecutionException, TimeoutException { From 2aee6fe9584650696d75b81c3b4e1feb844207b9 Mon Sep 17 00:00:00 2001 From: jaumeortola Date: Thu, 11 Apr 2024 17:00:24 +0200 Subject: [PATCH 2/3] fix: return true --- .../src/main/java/org/languagetool/JLanguageTool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/languagetool-core/src/main/java/org/languagetool/JLanguageTool.java b/languagetool-core/src/main/java/org/languagetool/JLanguageTool.java index 321f452d1a07..d496131b6bec 100644 --- a/languagetool-core/src/main/java/org/languagetool/JLanguageTool.java +++ b/languagetool-core/src/main/java/org/languagetool/JLanguageTool.java @@ -1093,7 +1093,7 @@ private boolean isRuleActiveForLanguageWithModel(Rule rule, Language language, M } } } - return false; + return true; } private final Map ruleSetCache = new ConcurrentHashMap<>(); From 92e903f4bc931eab4f18dab10709f5a17fd44a41 Mon Sep 17 00:00:00 2001 From: jaumeortola Date: Thu, 11 Apr 2024 17:15:11 +0200 Subject: [PATCH 3/3] [fr] add one more rule to disabled rules --- .../src/main/java/org/languagetool/JLanguageTool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/languagetool-core/src/main/java/org/languagetool/JLanguageTool.java b/languagetool-core/src/main/java/org/languagetool/JLanguageTool.java index d496131b6bec..6bf7fb8dd2cd 100644 --- a/languagetool-core/src/main/java/org/languagetool/JLanguageTool.java +++ b/languagetool-core/src/main/java/org/languagetool/JLanguageTool.java @@ -1085,7 +1085,7 @@ private List filterMatches(AnnotatedText annotatedText, RuleSet rules private boolean isRuleActiveForLanguageWithModel(Rule rule, Language language, Map remoteRulesResults) { if (language.getShortCode().equals("fr")) { - List disableFrenchRules = Arrays.asList("OU"); + List disableFrenchRules = Arrays.asList("OU", "OU_FIGEES"); RemoteRuleResult remoteRulesResult = remoteRulesResults.get("AI_FR_GGEC"); if (remoteRulesResult != null) { if (remoteRulesResult.isSuccess()) {