Inspect bugfix

avidtools/connectors/garak.py

@@ -10,6 +10,7 @@
 from urllib.error import HTTPError, URLError
 from urllib.request import Request, urlopen
 
+from .inspect import _resolve_parties_from_model
 from .utils import (
     apply_normalizations,
     choose_model_subject_label,
@@ -616,52 +617,70 @@ async def summarize_one(probe_name: str):
     return probe_cache, module_cache
 
 
-def _shorten_artifact_model_names(report: dict) -> Optional[str]:
+def _apply_party_normalization(report: dict) -> Optional[str]:
+    """Normalize developer, deployer, and artifact names.
+
+    For Together AI deployments (deployer 'together' or 'litellm'), the full
+    model path is reconstructed and passed to ``_resolve_parties_from_model``
+    so that human-readable developer and deployer names are derived alongside
+    the shortened model artifact name.
+
+    For all other models the artifact name is shortened to its last path
+    component and the deployer field is left unchanged.
+
+    Returns the preferred (shortened) model artifact name.
+    """
     affects = report.setdefault("affects", {})
+    deployer_raw = to_list(affects.get("deployer"))
     artifacts = affects.get("artifacts")
     if not isinstance(artifacts, list):
         return None
 
-    preferred_model = None
+    _together_deployers = {"together", "litellm"}
+    is_together = any(
+        v.strip().lower() in _together_deployers for v in deployer_raw
+    )
+
+    preferred_model: Optional[str] = None
+    dev_names: list = []
+    dep_names: list = []
+    seen_dev: set = set()
+    seen_dep: set = set()
+
     for artifact in artifacts:
         if not isinstance(artifact, dict):
             continue
         name = artifact.get("name")
         if not isinstance(name, str):
             continue
 
-        shortened = name.split("/", 1)[1] if "/" in name else name
-        artifact["name"] = shortened
-        if preferred_model is None:
-            preferred_model = shortened
-
-    return preferred_model
+        if is_together and "/" in name:
+            full_path = f"together/{name}"
+            developer_name, deployer_name, model_name = (
+                _resolve_parties_from_model(full_path)
+            )
+            if developer_name.lower() not in seen_dev:
+                seen_dev.add(developer_name.lower())
+                dev_names.append(developer_name)
+            if deployer_name.lower() not in seen_dep:
+                seen_dep.add(deployer_name.lower())
+                dep_names.append(deployer_name)
+        else:
+            model_name = name.split("/", 1)[1] if "/" in name else name
 
+        artifact["name"] = model_name
+        if preferred_model is None:
+            preferred_model = model_name
 
-def _apply_litellm_deployer_mapping(report: dict):
-    affects = report.setdefault("affects", {})
-    deployer = to_list(affects.get("deployer"))
+    if dev_names:
+        affects["developer"] = dev_names
+    if dep_names:
+        affects["deployer"] = dep_names
+    elif is_together and not dep_names:
+        # fallback if no artifacts with org/model pattern were found
+        affects["deployer"] = ["Together AI"]
 
-    mapped = []
-    changed = False
-    for value in deployer:
-        normalized = value.strip().lower()
-        if normalized == "litellm" or normalized == "together":
-            mapped.append("Together AI")
-            changed = True
-        else:
-            mapped.append(value)
-
-    if changed:
-        deduped = []
-        seen = set()
-        for value in mapped:
-            key = value.lower()
-            if key in seen:
-                continue
-            seen.add(key)
-            deduped.append(value)
-        affects["deployer"] = deduped
+    return preferred_model
 
 
 def _rebuild_text_descriptions(
@@ -771,8 +790,7 @@ def _normalize_report(
 ):
     """Apply Garak-specific normalize transforms to a single report."""
 
-    preferred_model_name = _shorten_artifact_model_names(report)
-    _apply_litellm_deployer_mapping(report)
+    preferred_model_name = _apply_party_normalization(report)
     apply_normalizations(
         report,
         preferred_model_name=preferred_model_name,

avidtools/connectors/inspect.py

@@ -2,9 +2,10 @@
 
 import json
 import re
+from datetime import datetime
 from html import unescape
 from pathlib import Path
-from typing import Any, List, Optional, Tuple
+from typing import Any, Iterable, List, Optional, Tuple
 from urllib.parse import quote
 from urllib.error import HTTPError, URLError
 from urllib.request import urlopen
@@ -44,6 +45,20 @@ def read_eval_log(file_path):
     "meta-llama": "Meta",
     "mistralai": "Mistral AI",
     "cohere": "Cohere",
+    "together": "Together AI",
+}
+
+_together_developer_name = {
+    "openai": "OpenAI",
+    "meta-llama": "Meta",
+    "mistralai": "Mistral AI",
+    "google": "Google",
+    "deepseek-ai": "DeepSeek",
+    "qwen": "Qwen",
+    "moonshotai": "Moonshot AI",
+    "minimaxai": "Minimax",
+    "liquidai": "Liquid AI",
+    "essentialai": "Essential AI",
 }
 
 SITE_ROOT = "https://ukgovernmentbeis.github.io/inspect_evals"
@@ -79,12 +94,36 @@ def import_eval_log(file_path: str) -> Any:
     return read_eval_log(file_path)
 
 
+def _resolve_parties_from_model(eval_model: str) -> Tuple[str, str, str]:
+    """Resolve developer, deployer, and artifact model name from eval model."""
+
+    if eval_model.startswith("together/"):
+        parts = eval_model.split("/")
+        if len(parts) >= 3:
+            together_dev = parts[1]
+            model_name = parts[-1]
+            developer_name = _together_developer_name.get(
+                together_dev.lower(),
+                together_dev,
+            )
+            return developer_name, "Together AI", model_name
+
+    model_prefix = eval_model.split("/", 1)[0]
+    developer_name = human_readable_name.get(
+        model_prefix,
+        model_prefix.replace("-", " ").title(),
+    )
+    model_name = eval_model.rsplit("/", 1)[-1]
+    return developer_name, eval_model, model_name
+
+
 def upload_eval_log_to_s3(
     file_path: str,
     bucket: str,
     key_prefix: str = "",
     region: Optional[str] = None,
     endpoint_url: Optional[str] = None,
+    skip_if_exists: bool = True,
 ) -> str:
     """Upload an Inspect eval log to S3 and return its URL."""
 
@@ -110,7 +149,13 @@ def upload_eval_log_to_s3(
         client_kwargs["endpoint_url"] = endpoint_url
 
     s3_client = boto3.client("s3", **client_kwargs)
-    s3_client.upload_file(str(source_path), bucket, key)
+    if skip_if_exists:
+        try:
+            s3_client.head_object(Bucket=bucket, Key=key)
+        except Exception:
+            s3_client.upload_file(str(source_path), bucket, key)
+    else:
+        s3_client.upload_file(str(source_path), bucket, key)
 
     quoted_key = quote(key, safe="/")
     if endpoint_url:
@@ -120,6 +165,46 @@ def upload_eval_log_to_s3(
     return f"https://{bucket}.s3.amazonaws.com/{quoted_key}"
 
 
+def _report_payload(report: Report) -> dict:
+    if hasattr(report, "model_dump"):
+        payload = report.model_dump(mode="json")
+    else:
+        payload = report.dict()
+
+    metrics = payload.get("metrics")
+    if isinstance(metrics, list):
+        flat_metrics = []
+        for metric in metrics:
+            if not isinstance(metric, dict):
+                continue
+            detection = metric.get("detection_method", {})
+            results = metric.get("results", {})
+            scorer = results.get("scorer") or detection.get("name")
+            flat_metrics.append(
+                {
+                    "scorer": scorer,
+                    "metrics": metric.get("name"),
+                    "value": results.get("value"),
+                }
+            )
+        payload["metrics"] = flat_metrics
+
+    return payload
+
+
+def write_reports_jsonl(reports: Iterable[Report], output_path: Path) -> int:
+    """Write reports to a JSONL file and return count written."""
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    count = 0
+    with output_path.open("w", encoding="utf-8") as file_obj:
+        for report in reports:
+            file_obj.write(json.dumps(_report_payload(report)))
+            file_obj.write("\n")
+            count += 1
+    return count
+
+
 def convert_eval_log(
     file_path: str,
     normalize: bool = False,
@@ -164,17 +249,14 @@ def convert_eval_log(
             endpoint_url=s3_endpoint_url,
         )
 
-    report = Report()
-    model_prefix = eval_log.eval.model.split("/", 1)[0]
-    developer_name = human_readable_name.get(
-        model_prefix,
-        model_prefix.replace("-", " ").title(),
+    report = Report(data_version="0.3.1")
+    developer_name, deployer_name, model_name = _resolve_parties_from_model(
+        eval_log.eval.model
     )
     task = eval_log.eval.task.rsplit("/", 1)[-1]
-    model_name = eval_log.eval.model.rsplit("/", 1)[-1]
     report.affects = Affects(
         developer=[developer_name],
-        deployer=[eval_log.eval.model],
+        deployer=[deployer_name],
         artifacts=[Artifact(type=ArtifactTypeEnum.model, name=model_name)],
     )
 
@@ -188,9 +270,8 @@ def convert_eval_log(
         description=LangValue(lang="eng", value=description_value),
     )
 
-    dataset_label = (
-        f"Inspect Evaluation Log for dataset: {eval_log.eval.dataset.name}"
-    )
+    dataset_name = getattr(eval_log.eval.dataset, "name", None) or task
+    dataset_label = f"Inspect Evaluation Log for dataset: {dataset_name}"
     dataset_location = (
         eval_log.eval.dataset.location
         if getattr(eval_log.eval.dataset, "location", None)
@@ -250,6 +331,15 @@ def convert_eval_log(
     )
     report.description = LangValue(lang="eng", value=full_description)
 
+    completed_at = getattr(eval_log.stats, "completed_at", None)
+    if completed_at:
+        try:
+            report.reported_date = datetime.fromisoformat(
+                str(completed_at)
+            ).date()
+        except (ValueError, TypeError):
+            pass
+
     if normalize:
         report_payload = (
             report.model_dump()
@@ -262,6 +352,30 @@ def convert_eval_log(
     return [report]
 
 
+def convert_eval_logs(
+    file_paths: Iterable[Path],
+    normalize: bool = False,
+    s3_bucket: Optional[str] = None,
+    s3_key_prefix: str = "",
+    s3_region: Optional[str] = None,
+    s3_endpoint_url: Optional[str] = None,
+) -> List[Report]:
+    """Convert multiple Inspect eval logs into AVID reports."""
+
+    all_reports: List[Report] = []
+    for file_path in file_paths:
+        reports = convert_eval_log(
+            str(file_path),
+            normalize=normalize,
+            s3_bucket=s3_bucket,
+            s3_key_prefix=s3_key_prefix,
+            s3_region=s3_region,
+            s3_endpoint_url=s3_endpoint_url,
+        )
+        all_reports.extend(reports)
+    return all_reports
+
+
 def _clean_html_to_text(fragment: str) -> str:
     """Strip HTML tags and normalize whitespace for section extraction."""
 
@@ -444,6 +558,8 @@ def _first_line(text: str) -> str:
 def normalize_report_data(report: dict):
     """Apply Inspect normalize transformations to a report dictionary."""
 
+    report.setdefault("data_version", "0.3.1")
+
     problem_desc = (
         report.get("problemtype", {})
         .get("description", {})