Updating scores for entity classifier for context matching (daxa-ai#210)

* Updating scores for entity classifier context matching
KumarNitin19 · Feb 23, 2024 · 2285ffe · 2285ffe
1 parent a5540d5
commit 2285ffe
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 5 deletions.
diff --git a/pebblo/entity_classifier/entity_classifier.py b/pebblo/entity_classifier/entity_classifier.py
@@ -93,10 +93,10 @@ def presidio_secret_classifier(self, input_text):
                 registry=custom_registry,
                 context_aware_enhancer=LemmaContextAwareEnhancer(
                     context_similarity_factor=float(
-                        ConfidenceScore.ContextSimilarityScore.value
+                        ConfidenceScore.EntityContextSimilarityFactor.value
                     ),
                     min_score_with_context_similarity=float(
-                        ConfidenceScore.Entity.value
+                        ConfidenceScore.EntityMinScoreWithContext.value
                     ),
                 ),
             )

diff --git a/pebblo/entity_classifier/utils/config.py b/pebblo/entity_classifier/utils/config.py
@@ -39,6 +39,9 @@ class SecretEntities(Enum):
 
 
 class ConfidenceScore(Enum):
-    Entity = "0.35"  # It denotes minimum confidence score when context matches
-    EntityMinScore = "0.4"  # It denotes the pattern's strength
-    ContextSimilarityScore = "0.3"  # It denotes how much to enhance confidence of match entity when context matches
+    Entity = "0.8"  # based on this score entity output is finalized
+    EntityMinScore = "0.45"  # It denotes the pattern's strength
+    EntityContextSimilarityFactor = (
+        "0.35"  # It denotes how much to enhance confidence of match entity
+    )
+    EntityMinScoreWithContext = "0.4"  # It denotes minimum confidence score