Add string operator $replaceAll/$replaceOne tests

documentdb_tests/compatibility/result_analyzer/analyzer.py

@@ -19,6 +19,8 @@
     "PASS": "passed",
     "FAIL": "failed",
     "SKIPPED": "skipped",
+    "XFAIL": "xfailed",
+    "XPASS": "xpassed",
 }
 
 
@@ -28,6 +30,8 @@ class TestOutcome:
     PASS = "PASS"
     FAIL = "FAIL"
     SKIPPED = "SKIPPED"
+    XFAIL = "XFAIL"
+    XPASS = "XPASS"
 
 
 def categorize_outcome(test_result: Dict[str, Any]) -> str:
@@ -38,21 +42,26 @@ def categorize_outcome(test_result: Dict[str, Any]) -> str:
     - PASS: Test passed
     - FAIL: Test failed (for any reason)
     - SKIPPED: Test skipped
+    - XFAIL: Test expected to fail and did fail
+    - XPASS: Test expected to fail but passed
 
     Args:
         test_result: Test result dictionary from pytest JSON report
 
     Returns:
-        One of: PASS, FAIL, SKIPPED
+        One of: PASS, FAIL, SKIPPED, XFAIL, XPASS
     """
     outcome = test_result.get("outcome", "")
 
     if outcome == "passed":
         return TestOutcome.PASS
     elif outcome == "skipped":
         return TestOutcome.SKIPPED
+    elif outcome == "xfailed":
+        return TestOutcome.XFAIL
+    elif outcome == "xpassed":
+        return TestOutcome.XPASS
     else:
-        # Failed or any other outcome
         return TestOutcome.FAIL
 
 
@@ -94,6 +103,11 @@ def extract_failure_tag(test_result: Dict[str, Any]) -> str:
     crash_info = call_info.get("crash", {})
     crash_message = crash_info.get("message", "")
 
+    # Detect strict XPASS from longrepr
+    longrepr = call_info.get("longrepr", "")
+    if longrepr.startswith("[XPASS(strict)]"):
+        return "XPASS_STRICT"
+
     match = re.search(r"\[([A-Z_]+)\]", crash_message)
     if match:
         return match.group(1)
@@ -300,10 +314,12 @@ def analyze_results(self, json_report_path: str) -> Dict[str, Any]:
             "passed": 0,
             "failed": 0,
             "skipped": 0,
+            "xfailed": 0,
+            "xpassed": 0,
         }
 
         by_tag: Dict[str, Dict[str, int]] = defaultdict(
-            lambda: {"passed": 0, "failed": 0, "skipped": 0}
+            lambda: {"passed": 0, "failed": 0, "skipped": 0, "xfailed": 0, "xpassed": 0}
         )
 
         tests_details = []

documentdb_tests/compatibility/result_analyzer/report_generator.py

@@ -101,6 +101,8 @@ def generate_text_report(analysis: Dict[str, Any], output_path: str):
     lines.append(f"Passed:       {summary['passed']} ({summary['pass_rate']}%)")
     lines.append(f"Failed:       {summary['failed']}")
     lines.append(f"Skipped:      {summary['skipped']}")
+    lines.append(f"XFailed:      {summary['xfailed']}")
+    lines.append(f"XPassed:      {summary['xpassed']}")
     lines.append("")
 
     # Results by tag
@@ -141,6 +143,16 @@ def generate_text_report(analysis: Dict[str, Any], output_path: str):
         for test in skipped_tests:
             lines.append(f"  {test['name']}")
 
+    # XPASS warning
+    xpassed_tests = [t for t in analysis["tests"] if t["outcome"] == "XPASS"]
+    if xpassed_tests:
+        lines.append("")
+        lines.append(f"⚠ ERROR: {len(xpassed_tests)} test(s) unexpectedly passed (XPASS).")
+        lines.append("  With xfail_strict=true, these should appear as failures instead.")
+        lines.append("  If you see this, the test run may not have used pytest.ini.")
+        for test in xpassed_tests:
+            lines.append(f"    {test['name']}")
+
     lines.append("")
     lines.append("=" * 80)
 
@@ -163,6 +175,8 @@ def print_summary(analysis: Dict[str, Any]):
     print(f"Passed:  {summary['passed']} ({summary['pass_rate']}%)")
     print(f"Failed:  {summary['failed']}")
     print(f"Skipped: {summary['skipped']}")
+    print(f"XFailed: {summary['xfailed']}")
+    print(f"XPassed: {summary['xpassed']}")
     print("=" * 60)
 
     # Failed test counts by type
@@ -174,4 +188,12 @@ def print_summary(analysis: Dict[str, Any]):
         for ft in sorted(grouped):
             print(f"  {ft}: {len(grouped[ft])}")
 
+    if summary["xpassed"] > 0:
+        xpassed_tests = [t for t in analysis["tests"] if t["outcome"] == "XPASS"]
+        print(f"\n⚠ ERROR: {summary['xpassed']} test(s) unexpectedly passed (XPASS).")
+        print("  With xfail_strict=true, these should appear as failures instead.")
+        print("  If you see this, the test run may not have used pytest.ini.")
+        for t in xpassed_tests:
+            print(f"    {t['name']}")
+
     print("=" * 60 + "\n")