Reported batch results and discrepancies in audit report

.coveragerc

@@ -6,7 +6,7 @@ omit =
     scripts/*
 
 [report]
-fail_under = 98
+fail_under = 99
 precision = 2
 skip_covered = true
 show_missing = true

server/api/reports.py

@@ -13,7 +13,7 @@
 )
 from ..util.isoformat import isoformat
 from ..util.group_by import group_by
-from ..audit_math import supersimple, sampler_contest
+from ..audit_math import supersimple, sampler_contest, macro
 from ..api.rounds import (
     cvrs_for_contest,
     sampled_ballot_interpretations_to_cvrs,
@@ -151,19 +151,17 @@ def pretty_discrepancy(
         return ""
 
 
-def pretty_batch_results(batch: Batch, contest: Contest) -> str:
-    choice_votes = []
-    for choice in contest.choices:
-        choice_result = next(
-            (
-                result.result
-                for result in batch.results
-                if result.contest_choice_id == choice.id
-            ),
-            0,
+def pretty_choice_votes(
+    choice_votes: Dict[str, int], not_found: Optional[int] = None
+) -> str:
+    return "; ".join(
+        [f"{name}: {votes}" for name, votes in choice_votes.items()]
+        + (
+            [f"Ballots not found (counted for loser): {not_found}"]
+            if not_found is not None
+            else []
         )
-        choice_votes.append(f"{choice.name}: {choice_result}")
-    return "; ".join(choice_votes)
+    )
 
 
 def heading(heading: str):
@@ -311,19 +309,6 @@ def audit_board_rows(election: Election):
     return rows
 
 
-def pretty_choice_votes(
-    choice_votes: Dict[str, int], not_found: Optional[int] = None
-) -> str:
-    return "; ".join(
-        [f"{name}: {votes}" for name, votes in choice_votes.items()]
-        + (
-            [f"Ballots not found (counted for loser): {not_found}"]
-            if not_found is not None
-            else []
-        )
-    )
-
-
 def round_rows(election: Election):
     rows = [
         heading("ROUNDS"),
@@ -686,25 +671,70 @@ def sampled_batch_rows(election: Election, jurisdiction: Jurisdiction = None):
     # We only support one contest for batch audits
     assert len(list(election.contests)) == 1
     contest = list(election.contests)[0]
+
     rows.append(
         [
             "Jurisdiction Name",
             "Batch Name",
             "Ticket Numbers",
             "Audited?",
-            "Audit Result",
+            "Audit Results",
+            "Reported Results",
+            "Discrepancy",
         ]
     )
     for batch in batches:
+        reported_results = {
+            choice.name: batch.jurisdiction.batch_tallies[batch.name][contest.id][
+                choice.id
+            ]
+            for choice in contest.choices
+        }
+
+        audit_results = {
+            choice: next(
+                (
+                    result.result
+                    for result in batch.results
+                    if result.contest_choice_id == choice.id
+                ),
+                0,
+            )
+            for choice in contest.choices
+        }
+
+        is_audited = len(batch.results) > 0
+
+        error = (
+            macro.compute_error(
+                batch.jurisdiction.batch_tallies[batch.name],
+                {
+                    contest.id: {
+                        choice.id: result for choice, result in audit_results.items()
+                    }
+                },
+                sampler_contest.from_db_contest(contest),
+            )
+            if is_audited
+            else None
+        )
+
         rows.append(
             [
                 batch.jurisdiction.name,
                 batch.name,
                 pretty_batch_ticket_numbers(batch, round_id_to_num),
-                pretty_boolean(len(batch.results) > 0),
-                pretty_batch_results(batch, contest),
+                pretty_boolean(is_audited),
+                pretty_choice_votes(
+                    {choice.name: result for choice, result in audit_results.items()}
+                )
+                if is_audited
+                else "",
+                pretty_choice_votes(reported_results),
+                error["counted_as"] if error else "",
             ]
         )
+
     return rows
 
 

server/audit_math/macro.py

@@ -11,15 +11,20 @@
 publication).
 """
 from decimal import Decimal, ROUND_CEILING
-from typing import Dict, Tuple, Any
+from typing import Dict, Tuple, Any, TypedDict, Optional, List
 from .sampler_contest import Contest
 
 
+class BatchError(TypedDict):
+    counted_as: int
+    weighted_error: Decimal
+
+
 def compute_error(
     batch_results: Dict[str, Dict[str, int]],
     sampled_results: Dict[str, Dict[str, int]],
     contest: Contest,
-) -> Decimal:
+) -> Optional[BatchError]:
     """
     Computes the error in this batch
 
@@ -42,30 +47,35 @@ def compute_error(
         the maximum across-contest relative overstatement for batch p
     """
 
-    error = Decimal(0.0)
-    margins = contest.margins
-
     if contest.name not in batch_results:
-        return Decimal(0.0)
-
-    for winner in margins["winners"]:
-        for loser in margins["losers"]:
-            v_wp = batch_results[contest.name][winner]
-            v_lp = batch_results[contest.name][loser]
+        return None
 
-            a_wp = sampled_results[contest.name][winner]
-            a_lp = sampled_results[contest.name][loser]
+    def error_for_candidate_pair(winner, loser) -> Optional[BatchError]:
+        v_wp = batch_results[contest.name][winner]
+        v_lp = batch_results[contest.name][loser]
 
-            V_wl = contest.candidates[winner] - contest.candidates[loser]
-            if V_wl == 0:
-                return Decimal("inf")
+        a_wp = sampled_results[contest.name][winner]
+        a_lp = sampled_results[contest.name][loser]
 
-            e_pwl = Decimal((v_wp - v_lp) - (a_wp - a_lp)) / Decimal(V_wl)
+        V_wl = contest.candidates[winner] - contest.candidates[loser]
+        error = (v_wp - v_lp) - (a_wp - a_lp)
+        if error == 0:
+            return None
 
-            if e_pwl > error:
-                error = e_pwl
+        # Count negative errors (errors in favor of the winner) as 0 to be conservative
+        error = max(error, 0)
+        weighted_error = Decimal(error) / Decimal(V_wl) if V_wl > 0 else Decimal("inf")
+        return BatchError(counted_as=error, weighted_error=weighted_error)
 
-    return error
+    maybe_errors = [
+        error_for_candidate_pair(winner, loser)
+        for winner in contest.margins["winners"]
+        for loser in contest.margins["losers"]
+    ]
+    errors: List[BatchError] = [error for error in maybe_errors if error is not None]
+    if len(errors) == 0:
+        return None
+    return max(errors, key=lambda error: error["weighted_error"])
 
 
 def compute_max_error(
@@ -148,7 +158,10 @@ def compute_U(
     U = Decimal(0.0)
     for batch in reported_results:
         if batch in sample_results:
-            U += compute_error(reported_results[batch], sample_results[batch], contest)
+            error = compute_error(
+                reported_results[batch], sample_results[batch], contest
+            )
+            U += error["weighted_error"] if error else 0
         else:
             U += compute_max_error(reported_results[batch], contest)
 
@@ -275,7 +288,8 @@ def compute_risk(
     U = compute_U(reported_results, {}, contest)
 
     for batch in sample_results:
-        e_p = compute_error(reported_results[batch], sample_results[batch], contest)
+        error = compute_error(reported_results[batch], sample_results[batch], contest)
+        e_p = error["weighted_error"] if error else Decimal(0)
 
         u_p = compute_max_error(reported_results[batch], contest)
 

server/tests/batch_comparison/snapshots/snap_test_batch_comparison.py

@@ -32,11 +32,11 @@
 1,Contest 1,Targeted,6,Yes,0.0825517715,DATETIME,DATETIME,candidate 1: 1200; candidate 2: 600; candidate 3: 600\r
 \r
 ######## SAMPLED BATCHES ########\r
-Jurisdiction Name,Batch Name,Ticket Numbers,Audited?,Audit Result\r
-J1,Batch 1,"Round 1: 0.720194360819624066, 0.777128466487428756",Yes,candidate 1: 500; candidate 2: 250; candidate 3: 250\r
-J1,Batch 6,Round 1: 0.899217854763070950,Yes,candidate 1: 100; candidate 2: 50; candidate 3: 50\r
-J1,Batch 8,Round 1: 0.9723790677174592551,Yes,candidate 1: 100; candidate 2: 50; candidate 3: 50\r
-J2,Batch 3,"Round 1: 0.368061935896261076, 0.733615858338543383",Yes,candidate 1: 500; candidate 2: 250; candidate 3: 250\r
+Jurisdiction Name,Batch Name,Ticket Numbers,Audited?,Audit Results,Reported Results,Discrepancy\r
+J1,Batch 1,"Round 1: 0.720194360819624066, 0.777128466487428756",Yes,candidate 1: 500; candidate 2: 250; candidate 3: 250,candidate 1: 500; candidate 2: 250; candidate 3: 250,\r
+J1,Batch 6,Round 1: 0.899217854763070950,Yes,candidate 1: 100; candidate 2: 50; candidate 3: 50,candidate 1: 100; candidate 2: 50; candidate 3: 50,\r
+J1,Batch 8,Round 1: 0.9723790677174592551,Yes,candidate 1: 100; candidate 2: 50; candidate 3: 50,candidate 1: 100; candidate 2: 50; candidate 3: 50,\r
+J2,Batch 3,"Round 1: 0.368061935896261076, 0.733615858338543383",Yes,candidate 1: 500; candidate 2: 250; candidate 3: 250,candidate 1: 500; candidate 2: 250; candidate 3: 250,\r
 """
 
 snapshots["test_batch_comparison_round_1 1"] = {
@@ -138,25 +138,25 @@
 2,Contest 1,Targeted,5,No,,DATETIME,,candidate 1: 0; candidate 2: 0; candidate 3: 0\r
 \r
 ######## SAMPLED BATCHES ########\r
-Jurisdiction Name,Batch Name,Ticket Numbers,Audited?,Audit Result\r
-J1,Batch 1,"Round 1: 0.720194360819624066, 0.777128466487428756",Yes,candidate 1: 400; candidate 2: 50; candidate 3: 40\r
-J1,Batch 6,Round 1: 0.899217854763070950,Yes,candidate 1: 100; candidate 2: 50; candidate 3: 40\r
-J1,Batch 8,Round 1: 0.9723790677174592551,Yes,candidate 1: 100; candidate 2: 50; candidate 3: 40\r
-J2,Batch 3,"Round 1: 0.368061935896261076, 0.733615858338543383",Yes,candidate 1: 100; candidate 2: 100; candidate 3: 40\r
-J1,Batch 3,Round 2: 0.753710009967479876,No,candidate 1: 0; candidate 2: 0; candidate 3: 0\r
-J1,Batch 4,"Round 2: 0.9553762217707628661, 0.9782132493451071914",No,candidate 1: 0; candidate 2: 0; candidate 3: 0\r
-J2,Batch 4,"Round 2: 0.608147659546583410, 0.868820918994249069",No,candidate 1: 0; candidate 2: 0; candidate 3: 0\r
+Jurisdiction Name,Batch Name,Ticket Numbers,Audited?,Audit Results,Reported Results,Discrepancy\r
+J1,Batch 1,"Round 1: 0.720194360819624066, 0.777128466487428756",Yes,candidate 1: 400; candidate 2: 50; candidate 3: 40,candidate 1: 500; candidate 2: 250; candidate 3: 250,0\r
+J1,Batch 6,Round 1: 0.899217854763070950,Yes,candidate 1: 100; candidate 2: 50; candidate 3: 40,candidate 1: 100; candidate 2: 50; candidate 3: 50,0\r
+J1,Batch 8,Round 1: 0.9723790677174592551,Yes,candidate 1: 100; candidate 2: 50; candidate 3: 40,candidate 1: 100; candidate 2: 50; candidate 3: 50,0\r
+J2,Batch 3,"Round 1: 0.368061935896261076, 0.733615858338543383",Yes,candidate 1: 100; candidate 2: 100; candidate 3: 40,candidate 1: 500; candidate 2: 250; candidate 3: 250,250\r
+J1,Batch 3,Round 2: 0.753710009967479876,No,,candidate 1: 500; candidate 2: 250; candidate 3: 250,\r
+J1,Batch 4,"Round 2: 0.9553762217707628661, 0.9782132493451071914",No,,candidate 1: 500; candidate 2: 250; candidate 3: 250,\r
+J2,Batch 4,"Round 2: 0.608147659546583410, 0.868820918994249069",No,,candidate 1: 500; candidate 2: 250; candidate 3: 250,\r
 """
 
 snapshots[
     "test_batch_comparison_round_2 9"
 ] = """######## SAMPLED BATCHES ########\r
-Jurisdiction Name,Batch Name,Ticket Numbers,Audited?,Audit Result\r
-J1,Batch 1,"Round 1: 0.720194360819624066, 0.777128466487428756",Yes,candidate 1: 400; candidate 2: 50; candidate 3: 40\r
-J1,Batch 6,Round 1: 0.899217854763070950,Yes,candidate 1: 100; candidate 2: 50; candidate 3: 40\r
-J1,Batch 8,Round 1: 0.9723790677174592551,Yes,candidate 1: 100; candidate 2: 50; candidate 3: 40\r
-J1,Batch 3,Round 2: 0.753710009967479876,No,candidate 1: 0; candidate 2: 0; candidate 3: 0\r
-J1,Batch 4,"Round 2: 0.9553762217707628661, 0.9782132493451071914",No,candidate 1: 0; candidate 2: 0; candidate 3: 0\r
+Jurisdiction Name,Batch Name,Ticket Numbers,Audited?,Audit Results,Reported Results,Discrepancy\r
+J1,Batch 1,"Round 1: 0.720194360819624066, 0.777128466487428756",Yes,candidate 1: 400; candidate 2: 50; candidate 3: 40,candidate 1: 500; candidate 2: 250; candidate 3: 250,0\r
+J1,Batch 6,Round 1: 0.899217854763070950,Yes,candidate 1: 100; candidate 2: 50; candidate 3: 40,candidate 1: 100; candidate 2: 50; candidate 3: 50,0\r
+J1,Batch 8,Round 1: 0.9723790677174592551,Yes,candidate 1: 100; candidate 2: 50; candidate 3: 40,candidate 1: 100; candidate 2: 50; candidate 3: 50,0\r
+J1,Batch 3,Round 2: 0.753710009967479876,No,,candidate 1: 500; candidate 2: 250; candidate 3: 250,\r
+J1,Batch 4,"Round 2: 0.9553762217707628661, 0.9782132493451071914",No,,candidate 1: 500; candidate 2: 250; candidate 3: 250,\r
 """
 
 snapshots["test_batch_comparison_sample_size 1"] = [

server/tests/batch_comparison/snapshots/snap_test_batches.py

@@ -27,11 +27,11 @@
 snapshots[
     "test_batches_human_sort_order 2"
 ] = """######## SAMPLED BATCHES ########\r
-Jurisdiction Name,Batch Name,Ticket Numbers,Audited?,Audit Result\r
-J1,Batch 1 - 1,"Round 1: 0.9610467367288398089, 0.9743784458526487453",No,candidate 1: 0; candidate 2: 0; candidate 3: 0\r
-J1,Batch 1 - 10,Round 1: 0.109576900310237874,No,candidate 1: 0; candidate 2: 0; candidate 3: 0\r
-J1,Batch 2,"Round 1: 0.474971525750860236, 0.555845039101209884",No,candidate 1: 0; candidate 2: 0; candidate 3: 0\r
-J1,Batch 10,"Round 1: 0.772049767819343419, 0.875085546411266410",No,candidate 1: 0; candidate 2: 0; candidate 3: 0\r
+Jurisdiction Name,Batch Name,Ticket Numbers,Audited?,Audit Results,Reported Results,Discrepancy\r
+J1,Batch 1 - 1,"Round 1: 0.9610467367288398089, 0.9743784458526487453",No,,candidate 1: 10; candidate 2: 5; candidate 3: 5,\r
+J1,Batch 1 - 10,Round 1: 0.109576900310237874,No,,candidate 1: 10; candidate 2: 5; candidate 3: 5,\r
+J1,Batch 2,"Round 1: 0.474971525750860236, 0.555845039101209884",No,,candidate 1: 10; candidate 2: 5; candidate 3: 5,\r
+J1,Batch 10,"Round 1: 0.772049767819343419, 0.875085546411266410",No,,candidate 1: 10; candidate 2: 5; candidate 3: 5,\r
 """
 
 snapshots["test_record_batch_results 1"] = {