Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions ami/main/api/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1942,36 +1942,36 @@ class ModelAgreementSerializer(serializers.Serializer):
max_value=1.0,
allow_null=True,
required=False,
help_text="Wilson 95% CI lower bound for agreed_exact_pct. Null when verified_with_prediction_count is 0.",
help_text="Wilson 95% CI lower bound for agreed_exact_pct. Null when comparable_count is 0.",
)
agreed_exact_ci_high = serializers.FloatField(
min_value=0.0,
max_value=1.0,
allow_null=True,
required=False,
help_text="Wilson 95% CI upper bound for agreed_exact_pct. Null when verified_with_prediction_count is 0.",
help_text="Wilson 95% CI upper bound for agreed_exact_pct. Null when comparable_count is 0.",
)
agreed_any_rank_count = serializers.IntegerField(
help_text="Exact matches plus disagreements whose LCA is at any real rank (UNKNOWN excluded)."
)
agreed_any_rank_pct = serializers.FloatField(
min_value=0.0,
max_value=1.0,
help_text="agreed_any_rank_count / verified_with_prediction_count",
help_text="agreed_any_rank_count / comparable_count",
)
agreed_any_rank_ci_low = serializers.FloatField(
min_value=0.0,
max_value=1.0,
allow_null=True,
required=False,
help_text="Wilson 95% CI lower bound for agreed_any_rank_pct. Null when verified_with_prediction_count is 0.",
help_text="Wilson 95% CI lower bound for agreed_any_rank_pct. Null when comparable_count is 0.",
)
agreed_any_rank_ci_high = serializers.FloatField(
min_value=0.0,
max_value=1.0,
allow_null=True,
required=False,
help_text="Wilson 95% CI upper bound for agreed_any_rank_pct. Null when verified_with_prediction_count is 0.",
help_text="Wilson 95% CI upper bound for agreed_any_rank_pct. Null when comparable_count is 0.",
)
cohens_kappa = serializers.FloatField(
min_value=-1.0,
Expand Down Expand Up @@ -2002,5 +2002,5 @@ class ModelAgreementSerializer(serializers.Serializer):
max_value=1.0,
allow_null=True,
required=False,
help_text="agreed_coarser_rank_count / verified_with_prediction_count. Null when no threshold supplied.",
help_text="agreed_coarser_rank_count / comparable_count. Null when no threshold supplied.",
)
22 changes: 22 additions & 0 deletions ami/main/tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -6020,6 +6020,28 @@ def test_taxon_less_verification_excluded_from_denominator(self):
# 1 exact agreement out of 1 comparable → 100%, not 50%.
self.assertEqual(result["agreed_exact_count"], 1)
self.assertAlmostEqual(result["agreed_exact_pct"], 1.0)
self.assertAlmostEqual(result["agreed_any_rank_pct"], 1.0)

def test_confidence_intervals_null_when_nothing_is_comparable(self):
"""Wilson bounds are null once nothing is comparable, even though every
verified occurrence here carries a machine prediction.

comparable_count falls below verified_with_prediction_count whenever a
verification has no taxon, and the intervals key off the former.
"""
from ami.main.models_future.occurrence import model_agreement_for_project

occurrences = list(Occurrence.objects.filter(project=self.project).order_by("pk"))
for occurrence in occurrences[:2]:
Identification.objects.create(user=self.user, occurrence=occurrence, taxon=None)

result = model_agreement_for_project(Occurrence.objects.filter(project=self.project))
self.assertEqual(result["verified_with_prediction_count"], 2)
self.assertEqual(result["comparable_count"], 0)
self.assertIsNone(result["agreed_exact_ci_low"])
self.assertIsNone(result["agreed_exact_ci_high"])
self.assertIsNone(result["agreed_any_rank_ci_low"])
self.assertIsNone(result["agreed_any_rank_ci_high"])


class TestOccurrenceStatsViewSet(APITestCase):
Expand Down
Loading