feat: add weighted confidence scoring for extraction validation

README.md

@@ -55,3 +55,34 @@ __Contributors:__
 - Vincent Harkins (@vharkins1)
 - Marc Vergés (@marcvergees) 
 - Jan Sans
+
+
+## Local Development Setup (Beginner Friendly)
+
+1. Clone your fork and enter project folder:
+
+ - git clone <your-fork-url>
+   cd FireForm (Terminal)
+
+2. Create virtual environment:
+
+ - python3 -m venv venv
+   source venv/bin/activate 
+
+3. Install dependencies:
+
+4. Initialize database tables:
+
+5. Run backend server:
+
+6. Open Swagger UI in browser: (http://127.0.0.1:8000/docs)
+
+### Common Errors
+
+- `sqlite3.OperationalError: no such table`
+→ Run database initialization step.
+
+- `Could not connect to Ollama`
+→ Ensure Ollama server is running locally.
+
+

api/db/models.py

@@ -15,4 +15,5 @@ class FormSubmission(SQLModel, table=True):
     template_id: int
     input_text: str
     output_pdf_path: str
-    created_at: datetime = Field(default_factory=datetime.utcnow)
+    requires_review: bool = False
+    created_at: datetime = Field(default_factory=datetime.utcnow)

src/controller.py

@@ -5,7 +5,14 @@ def __init__(self):
         self.file_manipulator = FileManipulator()
 
     def fill_form(self, user_input: str, fields: list, pdf_form_path: str):
-        return self.file_manipulator.fill_form(user_input, fields, pdf_form_path)
+        path, review_flag = self.file_manipulator.fill_form(
+        user_input=user_input,
+        fields=fields,
+        pdf_form_path=pdf_form_path
+    )
+        return path, review_flag
+
+
 
     def create_template(self, pdf_path: str):
         return self.file_manipulator.create_template(pdf_path)

src/schemas/incident-schema.py

@@ -0,0 +1,6 @@
+INCIDENT_SCHEMA = {
+    "location": "",
+    "time": "",
+    "severity": "",
+    "description": ""
+}

src/utils/extraction_validator.py

@@ -0,0 +1,28 @@
+class ExtractionValidator:
+    FIELD_WEIGHTS = {
+        "location": 30,
+        "time": 20,
+        "severity": 30,
+        "description": 20
+    }
+
+    def validate(self, data: dict):
+        missing_fields = []
+        confidence_score = 100
+
+        for field, weight in self.FIELD_WEIGHTS.items():
+            value = data.get(field)
+
+            if value is None or value == "" or value == "-1":
+                missing_fields.append(field)
+                confidence_score -= weight
+
+        confidence_score = max(confidence_score, 0)
+
+        requires_review = confidence_score < 70
+
+        return {
+            "requires_review": requires_review,
+            "missing_fields": missing_fields,
+            "confidence_score": confidence_score
+        }

src/utils/prompt_builder.py

@@ -0,0 +1,29 @@
+class PromptBuilder:
+    @staticmethod
+    def build_extraction_prompt(user_input: str, fields: list):
+        field_list = ", ".join(fields)
+
+        prompt = f"""
+You are an incident report extraction assistant.
+
+Extract the following fields strictly in JSON format:
+{field_list}
+
+Rules:
+- Return only valid JSON
+- Do not include explanations
+- Missing fields should be empty strings
+- Keep keys exactly as provided
+
+Incident description:
+{user_input}
+
+Expected format:
+{{
+    "location": "",
+    "time": "",
+    "severity": "",
+    "description": ""
+}}
+"""
+        return prompt

src/utils/validation.py

@@ -0,0 +1,11 @@
+def requires_review(data: dict, required_fields: list):
+    for field in required_fields:
+        value = data.get(field)
+
+        if value is None:
+            return True
+
+        if isinstance(value, str) and value.strip() in ["", "-1"]:
+            return True
+
+    return False

tests/test_extraction_validator.py

@@ -0,0 +1,78 @@
+from src.utils.extraction_validator import ExtractionValidator
+
+
+def test_missing_fields_requires_review():
+    validator = ExtractionValidator()
+
+    data = {
+        "location": "Bangalore",
+        "time": "",
+        "severity": "",
+        "description": ""
+    }
+
+    result = validator.validate(data)
+
+    assert result["requires_review"] is True
+    assert result["confidence_score"] < 100
+
+
+def test_complete_fields_no_review():
+    validator = ExtractionValidator()
+
+    data = {
+        "location": "Bangalore",
+        "time": "5 PM",
+        "severity": "High",
+        "description": "Fire on third floor"
+    }
+
+    result = validator.validate(data)
+
+    assert result["requires_review"] is False
+    assert result["confidence_score"] == 100
+
+
+def test_null_fields():
+    validator = ExtractionValidator()
+
+    data = {
+        "location": None,
+        "time": None,
+        "severity": None,
+        "description": None
+    }
+
+    result = validator.validate(data)
+
+    assert result["requires_review"] is True
+
+
+def test_placeholder_fields():
+    validator = ExtractionValidator()
+
+    data = {
+        "location": "-1",
+        "time": "-1",
+        "severity": "-1",
+        "description": "-1"
+    }
+
+    result = validator.validate(data)
+
+    assert result["requires_review"] is True
+
+def test_weighted_confidence_score():
+    validator = ExtractionValidator()
+
+    data = {
+        "location": "",
+        "time": "5 PM",
+        "severity": "",
+        "description": "Fire on third floor"
+    }
+
+    result = validator.validate(data)
+
+    assert result["confidence_score"] == 40
+    assert result["requires_review"] is True