{ "recorded_utc": "2026-10-23T20:42:23.062305+00:00", "scope": "T004 hosted v2 direct Activity negative behavior only", "active_version": "2", "agent_status": "active", "activity_requests_attempted": 2, "automatic_retries": 0, "auth_role_config_changes": false, "deployments": 0, "native_pass": false, "full_scenario_completion": false, "legal_completion": false, "original_v4_route_touched": false, "positive_reference": "hosted-repair-smoke.json", "source_sha256": { "retrieval.py": "35a2ceef21a880e104eeed53007d36cea8d51fcbc0af392c49dc09d4839a34f3", "main.py": "2d866a8616b363e844724b2c2fadf76fddcd5c1f2c57a16fdabf5257c78f3013", "evidence.py": "aa885de8512d0ba2ea7daa3eb0e5c8e26fc8eeabccf254526188a12874d97012" }, "expected_abstention": "Cannot determine from configured sources.", "invalid_guard": "len(question) > 12000; ValueError before model client construction", "invalid_guard_message": "A nonempty question of at most 12000 characters is required", "expected_application_error_pattern": "Source verification or delivery failed\\. No unverified answer was delivered\\. Reference: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", "ingress_identity": "Activity caller oid/tenant derived from actual in-memory Entra token claims", "cases": [ { "case": "no-relevant-evidence", "result": "FAIL", "http_status": 200, "activity_post_attempts": 1, "question_length": 67, "latency_ms": 27251.5, "session_log_read_exit_code": 0, "observed_downstream_responses_posts": 1, "actual_total_model_requests": null, "model_request_observability": "One application-level retrieval responses POST observed; internal retrieval-service model/tool request count is not exposed.", "envelope_accepted": true, "explicit_abstention": false, "explicit_application_error": true, "unexpected_application_error": true, "claimed_source_evidence_detected": false, "session_logs_correlated": true, "guard_valueerror_log_confirmed": false, "retrieval_response_log_confirmed": true, "model_invocation_excluded": false, "initial_assessment": "FAIL", "initial_session_log_read_exit_code": 1, "session_log_tail_limit": 300, "session_log_tail_limit_not_reached": true, "same_session_activity_request_count": 1, "reply_error_reference_matches_session_log": true, "private_response_sha256": "f7ec63d70e9f9604e018d20218e4879d022038c13c17744cae014dc5675458df", "private_session_log_sha256": "667f1fe35134edede564432238087f3d327e2beac60deb974dcca0a10fbce0ac", "selection_invoked": false, "reply_summary": "Source verification or delivery failed. No unverified answer was delivered. Reference: ", "no_false_claimed_source_evidence": true, "host_activity_latency_ms": 15195.3, "question": "What is the orbital period of Jupiter's moon Europa, in Earth days?", "diagnosis": "One downstream retrieval responses POST returned HTTP 200; retrieved_records failed at evidence.py json.loads(item['output']) with JSONDecodeError: Expecting value: line 2 column 1 (char 1). on_error returned explicit application verification error instead of required abstention. Selection was not reached.", "raw_retrieval_tool_output_observed": false }, { "case": "invalid-input", "result": "PASS", "http_status": 200, "activity_post_attempts": 1, "question_length": 12001, "latency_ms": 11053.05, "session_log_read_exit_code": 0, "observed_downstream_responses_posts": 0, "actual_total_model_requests": 0, "model_request_observability": "Runtime traceback confirms retrieval.py input guard raised before model client construction; one correlated Activity request, zero responses POSTs, no retrieval/selection completion.", "envelope_accepted": true, "explicit_abstention": false, "explicit_application_error": true, "unexpected_application_error": false, "claimed_source_evidence_detected": false, "session_logs_correlated": true, "guard_valueerror_log_confirmed": true, "retrieval_response_log_confirmed": false, "model_invocation_excluded": true, "initial_assessment": "FAIL", "initial_session_log_read_exit_code": 1, "session_log_tail_limit": 300, "session_log_tail_limit_not_reached": true, "same_session_activity_request_count": 1, "reply_error_reference_matches_session_log": true, "private_response_sha256": "c6503f645268717701a0838b23f8e33d5ce3116783ddaf376f1dd2a697a090ef", "private_session_log_sha256": "22c92a9040ead0a2bc706ec9ecbd9d358e5e6c95d18c890ebf0b7c0ef93baa79", "selection_invoked": false, "reply_summary": "Source verification or delivery failed. No unverified answer was delivered. Reference: ", "no_false_claimed_source_evidence": true, "host_activity_latency_ms": 2816.5, "diagnosis": "Oversized question rejected by actual application guard, explicit error returned without model invocation." } ], "completed_utc": "2026-10-23T20:43:04.168226+00:00", "result": "FAIL", "paid_request_accounting": { "authenticated_activity_posts": 2, "observed_downstream_responses_posts": 1, "observed_downstream_responses_posts_by_case": { "no-relevant-evidence": 1, "invalid-input": 0 }, "actual_total_model_requests": null, "invalid_input_actual_total_model_requests": 0, "billing_units_or_cost_verified": false, "note": "Two ingress Activity POSTs are separate from one observed downstream retrieval responses POST. Internal retrieval-service model/tool calls and actual billed units/cost are not independently observable." }, "diagnosis_completed_utc": "2026-10-23T20:44:21.081380+00:00", "activity_dispatch_command_exit_code": 0, "read_only_log_diagnosis_command_exit_code": 0, "original_initial_receipt_preserved": true, "log_collection_issue": "Initial read-only log commands exited 1: --tail 1000 exceeds azd's 300 limit. Original failed log reads and initial FAIL assessments preserved privately. Corrected read-only reads used --tail 300 for the same two sessions; both exited 0. No Activity requests repeated.", "acceptance_closed": false, "passed": 1, "failed": 1, "exclusive_dispatch_and_fresh_conversation_ids_verified": true, "full_envelope_and_actual_token_caller_metadata_verified": true }