Add frequency penalty and complete the final calculation

pom.xml

@@ -16,7 +16,7 @@
         <skipITs>true</skipITs>
         <surefire-plugin.version>3.5.3</surefire-plugin.version>
 
-        <podmortem.common.lib.version>1.0-c839aae-SNAPSHOT</podmortem.common.lib.version>
+        <podmortem.common.lib.version>1.0-13ea9c7-SNAPSHOT</podmortem.common.lib.version>
     </properties>
 
     <repositories>

src/main/java/com/redhat/podmortem/parser/service/FrequencyTrackingService.java

@@ -0,0 +1,154 @@
+package com.redhat.podmortem.parser.service;
+
+import com.redhat.podmortem.common.model.analysis.PatternFrequency;
+import jakarta.enterprise.context.ApplicationScoped;
+import java.time.Duration;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@ApplicationScoped
+public class FrequencyTrackingService {
+
+    private static final Logger log = LoggerFactory.getLogger(FrequencyTrackingService.class);
+
+    private final Map<String, PatternFrequency> patternFrequencies = new ConcurrentHashMap<>();
+
+    @ConfigProperty(name = "scoring.frequency.threshold", defaultValue = "10.0")
+    double frequencyThreshold;
+
+    @ConfigProperty(name = "scoring.frequency.max-penalty", defaultValue = "0.8")
+    double maxPenalty;
+
+    @ConfigProperty(name = "scoring.frequency.time-window-hours", defaultValue = "1")
+    int timeWindowHours;
+
+    /**
+     * Records a pattern match for frequency tracking.
+     *
+     * @param patternId The ID of the pattern that was matched
+     */
+    public void recordPatternMatch(String patternId) {
+        if (patternId == null || patternId.trim().isEmpty()) {
+            return;
+        }
+
+        PatternFrequency frequency =
+                patternFrequencies.computeIfAbsent(
+                        patternId, k -> new PatternFrequency(Duration.ofHours(timeWindowHours)));
+
+        frequency.incrementCount();
+
+        log.debug(
+                "Recorded match for pattern '{}', current count: {}",
+                patternId,
+                frequency.getCurrentCount());
+    }
+
+    /**
+     * Calculates the frequency penalty for a given pattern.
+     *
+     * @param patternId The ID of the pattern
+     * @return Frequency penalty between 0.0 (no penalty) and maxPenalty (maximum penalty)
+     */
+    public double calculateFrequencyPenalty(String patternId) {
+        if (patternId == null || patternId.trim().isEmpty()) {
+            return 0.0;
+        }
+
+        PatternFrequency frequency = patternFrequencies.get(patternId);
+        if (frequency == null) {
+            return 0.0; // no matches recorded yet, no penalty
+        }
+
+        double currentRate = frequency.getHourlyRate();
+
+        if (currentRate <= frequencyThreshold) {
+            return 0.0; // below threshold, no penalty
+        }
+
+        // calculate penalty as a percentage of how much we exceed the threshold
+        // penalty = min(maxPenalty, (currentRate - threshold) / threshold * scaleFactor)
+        double excessRate = currentRate - frequencyThreshold;
+        double penalty = Math.min(maxPenalty, excessRate / frequencyThreshold);
+
+        log.debug(
+                "Pattern '{}' frequency penalty: rate={}, threshold={}, penalty={}",
+                patternId,
+                currentRate,
+                frequencyThreshold,
+                penalty);
+
+        return penalty;
+    }
+
+    /**
+     * Gets the current frequency information for a pattern.
+     *
+     * @param patternId The ID of the pattern
+     * @return The PatternFrequency object, or null if not found
+     */
+    public PatternFrequency getPatternFrequency(String patternId) {
+        return patternFrequencies.get(patternId);
+    }
+
+    /**
+     * Gets current frequency statistics for all tracked patterns.
+     *
+     * @return Map of pattern IDs to their current match counts
+     */
+    public Map<String, Integer> getFrequencyStatistics() {
+        Map<String, Integer> stats = new ConcurrentHashMap<>();
+        patternFrequencies.forEach(
+                (patternId, frequency) -> stats.put(patternId, frequency.getCurrentCount()));
+        return stats;
+    }
+
+    /**
+     * Resets frequency tracking for a specific pattern.
+     *
+     * @param patternId The ID of the pattern to reset
+     */
+    public void resetPatternFrequency(String patternId) {
+        PatternFrequency frequency = patternFrequencies.get(patternId);
+        if (frequency != null) {
+            frequency.reset();
+            log.info("Reset frequency tracking for pattern '{}'", patternId);
+        }
+    }
+
+    /** Resets frequency tracking for all patterns. */
+    public void resetAllFrequencies() {
+        patternFrequencies.clear();
+        log.info("Reset frequency tracking for all patterns");
+    }
+
+    /**
+     * Gets the current frequency threshold.
+     *
+     * @return The frequency threshold
+     */
+    public double getFrequencyThreshold() {
+        return frequencyThreshold;
+    }
+
+    /**
+     * Gets the maximum penalty value.
+     *
+     * @return The maximum penalty
+     */
+    public double getMaxPenalty() {
+        return maxPenalty;
+    }
+
+    /**
+     * Gets the time window for frequency tracking.
+     *
+     * @return The time window in hours
+     */
+    public int getTimeWindowHours() {
+        return timeWindowHours;
+    }
+}

src/main/java/com/redhat/podmortem/parser/service/ScoringService.java

@@ -6,6 +6,7 @@
 import com.redhat.podmortem.common.model.pattern.SequenceEvent;
 import com.redhat.podmortem.common.model.pattern.SequencePattern;
 import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
 import java.util.List;
 import java.util.Map;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
@@ -32,6 +33,9 @@ public class ScoringService {
     @ConfigProperty(name = "scoring.proximity.max-window", defaultValue = "100")
     int maxWindow;
 
+    @Inject ContextAnalysisService contextAnalysisService;
+    @Inject FrequencyTrackingService frequencyTrackingService;
+
     /**
      * Calculates the final score for a matched event.
      *
@@ -53,16 +57,34 @@ public double calculateScore(MatchedEvent event, String[] allLines) {
         // calculate temporal factor using sequence pattern analysis
         double temporalFactor = calculateTemporalFactor(event, allLines);
 
+        // calculate context factor using keyword weighting
+        double contextFactor = contextAnalysisService.calculateContextFactor(event.getContext());
+
+        // calculate frequency penalty
+        double frequencyPenalty =
+                frequencyTrackingService.calculateFrequencyPenalty(pattern.getId());
+
+        // record this match for future frequency tracking
+        frequencyTrackingService.recordPatternMatch(pattern.getId());
+
         log.debug(
-                "Pattern '{}': Base Confidence={}, Severity Multiplier={}, Proximity Factor={}, Temporal Factor={}",
+                "Pattern '{}': Base Confidence={}, Severity Multiplier={}, Proximity Factor={}, Temporal Factor={}, Context Factor={}, Frequency Penalty={}",
                 pattern.getName(),
                 baseConfidence,
                 severityMultiplier,
                 proximityFactor,
-                temporalFactor);
-
-        // final score calculation using multiplicative formula
-        double finalScore = baseConfidence * severityMultiplier * proximityFactor * temporalFactor;
+                temporalFactor,
+                contextFactor,
+                frequencyPenalty);
+
+        // final score calculation
+        double finalScore =
+                baseConfidence
+                        * severityMultiplier
+                        * proximityFactor
+                        * temporalFactor
+                        * contextFactor
+                        * (1.0 - frequencyPenalty);
 
         return finalScore;
     }

src/main/resources/application.properties

@@ -8,3 +8,8 @@ scoring.proximity.max-window=100
 
 # Context keywords
 scoring.context.keywords-directory=keywords
+
+# Frequency tracking parameters for penalty calculation
+scoring.frequency.threshold=10.0
+scoring.frequency.max-penalty=0.8
+scoring.frequency.time-window-hours=1

src/main/resources/keywords/golang.json

@@ -22,4 +22,4 @@
     "heap": 0.1,
     "stack overflow": 0.4
   }
-} 
+}

src/main/resources/keywords/java-spring.json

@@ -62,4 +62,4 @@
     "security": 0.2,
     "denied": 0.2
   }
-} 
+}

src/main/resources/keywords/kubernetes.json

@@ -27,4 +27,4 @@
     "ConfigMap": 0.05,
     "Secret": 0.1
   }
-} 
+}

src/main/resources/keywords/python.json

@@ -36,4 +36,4 @@
     "timeout": 0.3,
     "blocked": 0.2
   }
-} 
+}