From 6b270b6eb82a946525300d9e6eb651e0d653d093 Mon Sep 17 00:00:00 2001 From: Garrett Maring Date: Tue, 26 May 2026 13:38:44 -0300 Subject: [PATCH] V41 Gate 8: integrate prompt benchmark telemetry report Add the package-backed V41PromptProgramBenchmarkReport, deterministic source-safe artifact, generator, checker, exports, and protocol tests. Bind V38 prompt benchmark and inference telemetry roots, V39 operational repair readback, V40 prompt benchmark smoke, and V41 Gate 2 through Gate 7 artifacts into post-rewrite benchmark telemetry proof. Update V41 specs, roadmap, docs, package scripts, and gate/canon workflow hooks; refresh dependent V41 artifacts for Gate 8 roadmap progression. --- ...rsation-tool-interface-prompt-rewrite.json | 28 +- .../v41-prompt-program-benchmark-report.json | 1875 +++++++++++++++++ .../v41-readfitsfinding-prompt-hardening.json | 22 +- .github/workflows/bitcode-canon-quality.yml | 3 + .github/workflows/bitcode-gate-quality.yml | 3 + BITCODE_SPEC_V41.md | 4 + BITCODE_SPEC_V41_DELTA.md | 2 + BITCODE_SPEC_V41_NOTES.md | 9 +- BITCODE_SPEC_V41_PARITY_MATRIX.md | 4 +- README.md | 9 + SPECIFICATIONS_ROADMAP.md | 5 +- package.json | 3 + packages/protocol/README.md | 12 + ...versation-tool-interface-prompt-rewrite.js | 4 +- .../v41-prompt-program-benchmark-report.js | 1010 +++++++++ .../v41-readfitsfinding-prompt-hardening.js | 4 +- packages/protocol/src/index.d.ts | 9 + packages/protocol/src/index.js | 11 + ...41-prompt-program-benchmark-report.test.js | 122 ++ ...1-gate1-prompt-program-roadmap-opening.mjs | 2 +- ...te4-reading-prompt-benchmark-baselines.mjs | 7 +- ...ck-v41-gate5-readneed-prompt-hardening.mjs | 11 +- ...gate6-readfitsfinding-prompt-hardening.mjs | 12 +- ...ersation-tool-interface-prompt-rewrite.mjs | 15 +- ...-gate8-prompt-program-benchmark-report.mjs | 299 +++ ...te-v41-prompt-program-benchmark-report.mjs | 62 + 26 files changed, 3499 insertions(+), 48 deletions(-) create mode 100644 .bitcode/v41-prompt-program-benchmark-report.json create mode 100644 packages/protocol/src/canonical/v41-prompt-program-benchmark-report.js create mode 100644 packages/protocol/test/v41-prompt-program-benchmark-report.test.js create mode 100644 scripts/check-v41-gate8-prompt-program-benchmark-report.mjs create mode 100644 scripts/generate-v41-prompt-program-benchmark-report.mjs diff --git a/.bitcode/v41-conversation-tool-interface-prompt-rewrite.json b/.bitcode/v41-conversation-tool-interface-prompt-rewrite.json index 27d25d372..9bd2313c8 100644 --- a/.bitcode/v41-conversation-tool-interface-prompt-rewrite.json +++ b/.bitcode/v41-conversation-tool-interface-prompt-rewrite.json @@ -6,7 +6,7 @@ "sourceSafetyVerdict": "source-safe-conversation-tool-interface-prompt-rewrite-metadata", "artifactPath": ".bitcode/v41-conversation-tool-interface-prompt-rewrite.json", "generatedAt": "2026-05-26T00:00:00.000Z", - "artifactRoot": "v41-conversation-tool-interface-prompt-rewrite:8acb0aa7e70efcf0b697d728", + "artifactRoot": "v41-conversation-tool-interface-prompt-rewrite:1c5fab85463bf1064cc5bce5", "passed": true, "failures": [], "metricIds": [ @@ -921,20 +921,20 @@ "passed": true, "hardeningScore": 1, "sourceHashes": { - "packages/protocol/src/canonical/v41-conversation-tool-interface-prompt-rewrite.js": "5403f09cd3ea720e8cd45908", + "packages/protocol/src/canonical/v41-conversation-tool-interface-prompt-rewrite.js": "3bf1de40f2dfd8f0d4e9be76", "packages/protocol/test/v41-conversation-tool-interface-prompt-rewrite.test.js": "0f36d8bf304ccc54f6eae328", "scripts/generate-v41-conversation-tool-interface-prompt-rewrite.mjs": "16b23544d652ac6d7e1afa81", - "scripts/check-v41-gate7-conversation-tool-interface-prompt-rewrite.mjs": "e092908fe6d7eeb486d919c0", - "BITCODE_SPEC_V41.md": "d8c0b4c7f8c268964304aed1", - "BITCODE_SPEC_V41_DELTA.md": "1d015ca6f1ee04481adc4d94", - "BITCODE_SPEC_V41_NOTES.md": "10535375dd3ba572d859cd96", - "BITCODE_SPEC_V41_PARITY_MATRIX.md": "00a808cdce6b067b3b74c284", - "SPECIFICATIONS_ROADMAP.md": "28927ec4df7539979189a250", - "README.md": "ea094a4cf3ceb26af0572f51", - "packages/protocol/README.md": "a350f4f057f5cfd5378e0bde", - "package.json": "9bd806a6dbff2448f0e5074e", - ".github/workflows/bitcode-gate-quality.yml": "07c24c49b8ab517a4426ed6b", - ".github/workflows/bitcode-canon-quality.yml": "36d6ce8158e1595685f7856a" + "scripts/check-v41-gate7-conversation-tool-interface-prompt-rewrite.mjs": "944c385a9d35dc0efaf024d1", + "BITCODE_SPEC_V41.md": "35ed9e60cecdcb8bd820caee", + "BITCODE_SPEC_V41_DELTA.md": "68092aae6b6f8916ee5652a0", + "BITCODE_SPEC_V41_NOTES.md": "7ed03aa76aa39be60d45ab0c", + "BITCODE_SPEC_V41_PARITY_MATRIX.md": "77e1bfa9c3dd2d549d5123e3", + "SPECIFICATIONS_ROADMAP.md": "ea44c2816a4548954f32a742", + "README.md": "64b27506f4fb734632abbb4f", + "packages/protocol/README.md": "b755f91f456d78b111c14794", + "package.json": "319d175fa7fedbc48b90cb37", + ".github/workflows/bitcode-gate-quality.yml": "4498ef474a10506a6cf781ce", + ".github/workflows/bitcode-canon-quality.yml": "75c13714eda5b1bf46ad896e" } } ], @@ -1306,7 +1306,7 @@ "gate3RegistryInterpolationRoot": "v41-registry-interpolation-contract:fb4b99ad86f0cf6f4a9c04c4", "gate4ReadingPromptBenchmarkBaselineRoot": "v41-reading-prompt-benchmark-baselines:82e956682fa1353ed9df6cff", "gate5ReadNeedPromptHardeningRoot": "v41-readneed-prompt-hardening:e93084e4403cc37d87fca2cf", - "gate6ReadFitsFindingPromptHardeningRoot": "v41-readfitsfinding-prompt-hardening:f5a01c099fd260fca0c081c8" + "gate6ReadFitsFindingPromptHardeningRoot": "v41-readfitsfinding-prompt-hardening:86e74080708372081079c021" }, "coverage": { "rowCount": 9, diff --git a/.bitcode/v41-prompt-program-benchmark-report.json b/.bitcode/v41-prompt-program-benchmark-report.json new file mode 100644 index 000000000..3962f9685 --- /dev/null +++ b/.bitcode/v41-prompt-program-benchmark-report.json @@ -0,0 +1,1875 @@ +{ + "artifactId": "v41-prompt-program-benchmark-report", + "schemaId": "bitcode.v41.promptProgramBenchmarkReport.v1", + "version": "V41", + "currentTarget": "V40", + "generatedAt": "2026-05-26T00:00:00.000Z", + "artifactPath": ".bitcode/v41-prompt-program-benchmark-report.json", + "artifactRoot": "v41-prompt-program-benchmark-report:6aa4e99080376c3bd24b72cf", + "sourceSafetyVerdict": "source-safe-prompt-program-benchmark-telemetry-metadata", + "metricIds": [ + "post_rewrite_promptpart_delta", + "post_rewrite_prompt_delta", + "benchmark_fixture_result_projection", + "prompt_lineage_registry_versioning", + "failsafe_thricified_receipt_projection", + "parsed_output_schema_verdict_projection", + "rich_stream_prompt_telemetry_projection", + "repair_hook_redaction_posture", + "source_safe_benchmark_report_disclosure", + "depository_search_embedding_query_projection" + ], + "disclosureTiers": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "prompt_registry_lineage_source_safe", + "benchmark_fixture_id_source_safe", + "benchmark_delta_score_source_safe", + "telemetry_event_identity_source_safe", + "failsafe_receipt_identity_source_safe", + "thricified_generation_identity_source_safe", + "parsed_output_shape_source_safe", + "schema_verdict_source_safe", + "repair_hook_identity_source_safe", + "source_hash_source_safe", + "raw_prompt_text_private", + "raw_interpolated_prompt_private", + "raw_provider_response_private", + "protected_prompt_private", + "protected_source_private", + "private_context_private", + "unpaid_assetpack_source_private", + "wallet_private_material_private", + "settlement_private_payload_private" + ], + "rows": [ + { + "reportRowId": "post-rewrite-prompt-inventory-delta", + "label": "Gate 2 prompt inventory deltas are re-bound after Reading and Conversation prompt rewrites", + "sourceRoots": [ + "packages/protocol/src/canonical/v41-promptpart-prompt-inventory.js", + ".bitcode/v41-promptpart-prompt-inventory.json", + ".bitcode/v41-readneed-prompt-hardening.json", + ".bitcode/v41-readfitsfinding-prompt-hardening.json", + ".bitcode/v41-conversation-tool-interface-prompt-rewrite.json" + ], + "promptProgramArtifactIds": [ + "v41-promptpart-prompt-inventory", + "v41-readneed-prompt-hardening", + "v41-readfitsfinding-prompt-hardening", + "v41-conversation-tool-interface-prompt-rewrite" + ], + "metricIds": [ + "post_rewrite_promptpart_delta", + "post_rewrite_prompt_delta", + "source_safe_benchmark_report_disclosure" + ], + "telemetryReceiptIds": [ + "promptpart_identity_source_safe", + "prompt_identity_source_safe" + ], + "requiredPredicateIds": [ + "gate2-inventory-passing", + "gate5-readneed-passing", + "gate6-readfitsfinding-passing", + "gate7-conversation-interface-passing", + "post-rewrite-artifacts-present", + "post-rewrite-report-does-not-serialize-raw-prompt-text" + ], + "rowRoot": "v41-prompt-program-benchmark-report-row:fc4db1968f444be5b895202a", + "benchmarkReportMode": "post-rewrite-source-safe-metadata", + "sourceSafetyClass": "source_safe_prompt_program_benchmark_telemetry_metadata", + "sourceSafeMetadataOnly": true, + "disclosureTiers": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "prompt_registry_lineage_source_safe", + "benchmark_fixture_id_source_safe", + "benchmark_delta_score_source_safe", + "telemetry_event_identity_source_safe", + "failsafe_receipt_identity_source_safe", + "thricified_generation_identity_source_safe", + "parsed_output_shape_source_safe", + "schema_verdict_source_safe", + "repair_hook_identity_source_safe", + "source_hash_source_safe", + "raw_prompt_text_private", + "raw_interpolated_prompt_private", + "raw_provider_response_private", + "protected_prompt_private", + "protected_source_private", + "private_context_private", + "unpaid_assetpack_source_private", + "wallet_private_material_private", + "settlement_private_payload_private" + ], + "rawPromptTextSerialized": false, + "rawInterpolatedPromptSerialized": false, + "rawProviderResponseSerialized": false, + "protectedPromptSerialized": false, + "protectedSourceVisible": false, + "privateContextSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "walletPrivateMaterialVisible": false, + "settlementPrivatePayloadVisible": false, + "forbiddenPayloadClasses": [ + "secret-values", + "provider-tokens", + "wallet-private-material", + "settlement-private-payloads", + "protected-source-payloads", + "raw-protected-prompts", + "raw-prompt-text", + "raw-interpolated-prompts", + "raw-provider-responses", + "private-context", + "unpaid-assetpack-source" + ], + "sourceStats": { + "sourceRootCount": 5, + "sourceRootPresentCount": 5, + "sourceFileCount": 5, + "sourceFileHashes": { + ".bitcode/v41-conversation-tool-interface-prompt-rewrite.json": "5d4736a363f57382f62d6a3d", + ".bitcode/v41-promptpart-prompt-inventory.json": "927738d4b3bdab4f2f9734e7", + ".bitcode/v41-readfitsfinding-prompt-hardening.json": "2b19151059efe156c6e135ac", + ".bitcode/v41-readneed-prompt-hardening.json": "148ce8c1c9bc8ebf03f1b2d2", + "packages/protocol/src/canonical/v41-promptpart-prompt-inventory.js": "75952a25cc72ee8b58d5a956" + } + }, + "requiredPredicateCount": 6, + "passedPredicateCount": 6, + "failedPredicateIds": [], + "passed": true + }, + { + "reportRowId": "readneed-post-rewrite-benchmark-delta", + "label": "ReadNeedComprehensionSynthesis benchmark deltas bind prompt revisions to typed Need outputs", + "sourceRoots": [ + "packages/protocol/src/canonical/v41-reading-prompt-benchmark-baselines.js", + ".bitcode/v41-reading-prompt-benchmark-baselines.json", + "packages/protocol/src/canonical/v41-readneed-prompt-hardening.js", + ".bitcode/v41-readneed-prompt-hardening.json", + "packages/pipelines/asset-pack/src/reading-pipeline-contract.ts", + "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts" + ], + "promptProgramArtifactIds": [ + "v41-reading-prompt-benchmark-baselines", + "v41-readneed-prompt-hardening" + ], + "metricIds": [ + "benchmark_fixture_result_projection", + "parsed_output_schema_verdict_projection", + "prompt_lineage_registry_versioning" + ], + "telemetryReceiptIds": [ + "ReadNeed", + "ReadNeedComprehensionSynthesisSchema", + "parsed_typed_output_shape_source_safe" + ], + "requiredPredicateIds": [ + "gate4-reading-benchmark-passing", + "readneed-hardening-row-count", + "readneed-benchmark-baselines-have-quality-scores", + "readneed-observability-projects-parsed-output", + "readneed-source-safe-metadata-only" + ], + "rowRoot": "v41-prompt-program-benchmark-report-row:5d4f8301fdbc3b0040f11e49", + "benchmarkReportMode": "post-rewrite-source-safe-metadata", + "sourceSafetyClass": "source_safe_prompt_program_benchmark_telemetry_metadata", + "sourceSafeMetadataOnly": true, + "disclosureTiers": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "prompt_registry_lineage_source_safe", + "benchmark_fixture_id_source_safe", + "benchmark_delta_score_source_safe", + "telemetry_event_identity_source_safe", + "failsafe_receipt_identity_source_safe", + "thricified_generation_identity_source_safe", + "parsed_output_shape_source_safe", + "schema_verdict_source_safe", + "repair_hook_identity_source_safe", + "source_hash_source_safe", + "raw_prompt_text_private", + "raw_interpolated_prompt_private", + "raw_provider_response_private", + "protected_prompt_private", + "protected_source_private", + "private_context_private", + "unpaid_assetpack_source_private", + "wallet_private_material_private", + "settlement_private_payload_private" + ], + "rawPromptTextSerialized": false, + "rawInterpolatedPromptSerialized": false, + "rawProviderResponseSerialized": false, + "protectedPromptSerialized": false, + "protectedSourceVisible": false, + "privateContextSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "walletPrivateMaterialVisible": false, + "settlementPrivatePayloadVisible": false, + "forbiddenPayloadClasses": [ + "secret-values", + "provider-tokens", + "wallet-private-material", + "settlement-private-payloads", + "protected-source-payloads", + "raw-protected-prompts", + "raw-prompt-text", + "raw-interpolated-prompts", + "raw-provider-responses", + "private-context", + "unpaid-assetpack-source" + ], + "sourceStats": { + "sourceRootCount": 6, + "sourceRootPresentCount": 6, + "sourceFileCount": 6, + "sourceFileHashes": { + ".bitcode/v41-reading-prompt-benchmark-baselines.json": "ee3f4a7261aeea5da1ccb95c", + ".bitcode/v41-readneed-prompt-hardening.json": "148ce8c1c9bc8ebf03f1b2d2", + "packages/pipelines/asset-pack/src/reading-pipeline-contract.ts": "b19eba649308640b1854dea6", + "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts": "3cc1dbd2a18b7db46440d0ba", + "packages/protocol/src/canonical/v41-reading-prompt-benchmark-baselines.js": "983d97655980b9a56e1481f0", + "packages/protocol/src/canonical/v41-readneed-prompt-hardening.js": "8b1eed4bd12fffb273f18711" + } + }, + "requiredPredicateCount": 5, + "passedPredicateCount": 5, + "failedPredicateIds": [], + "passed": true + }, + { + "reportRowId": "readfitsfinding-post-rewrite-benchmark-delta", + "label": "ReadFitsFindingSynthesis benchmark deltas bind many-candidate search, embeddings, and AssetPack preview metadata", + "sourceRoots": [ + "packages/protocol/src/canonical/v41-reading-prompt-benchmark-baselines.js", + ".bitcode/v41-reading-prompt-benchmark-baselines.json", + "packages/protocol/src/canonical/v41-readfitsfinding-prompt-hardening.js", + ".bitcode/v41-readfitsfinding-prompt-hardening.json", + "packages/protocol/src/canonical/read-fits-finding-search-embeddings.js", + "packages/pipelines/asset-pack/src/depository-search.ts", + "packages/pipelines/asset-pack/src/embedding-config.ts", + "packages/pipelines/asset-pack/src/read-fits-finding-runtime.ts" + ], + "promptProgramArtifactIds": [ + "v41-reading-prompt-benchmark-baselines", + "v41-readfitsfinding-prompt-hardening", + "v38-read-fits-finding-search-embeddings" + ], + "metricIds": [ + "benchmark_fixture_result_projection", + "depository_search_embedding_query_projection", + "parsed_output_schema_verdict_projection" + ], + "telemetryReceiptIds": [ + "ReadFitsFindingSynthesis", + "CandidateFits", + "AssetPackPreviewBoundary" + ], + "requiredPredicateIds": [ + "readfitsfinding-hardening-passing", + "readfitsfinding-has-many-candidate-search", + "readfitsfinding-embedding-config-present", + "readfitsfinding-search-embeddings-passing", + "readfitsfinding-source-safe-preview-only" + ], + "rowRoot": "v41-prompt-program-benchmark-report-row:94f6b8140b279cc1a41f5ed9", + "benchmarkReportMode": "post-rewrite-source-safe-metadata", + "sourceSafetyClass": "source_safe_prompt_program_benchmark_telemetry_metadata", + "sourceSafeMetadataOnly": true, + "disclosureTiers": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "prompt_registry_lineage_source_safe", + "benchmark_fixture_id_source_safe", + "benchmark_delta_score_source_safe", + "telemetry_event_identity_source_safe", + "failsafe_receipt_identity_source_safe", + "thricified_generation_identity_source_safe", + "parsed_output_shape_source_safe", + "schema_verdict_source_safe", + "repair_hook_identity_source_safe", + "source_hash_source_safe", + "raw_prompt_text_private", + "raw_interpolated_prompt_private", + "raw_provider_response_private", + "protected_prompt_private", + "protected_source_private", + "private_context_private", + "unpaid_assetpack_source_private", + "wallet_private_material_private", + "settlement_private_payload_private" + ], + "rawPromptTextSerialized": false, + "rawInterpolatedPromptSerialized": false, + "rawProviderResponseSerialized": false, + "protectedPromptSerialized": false, + "protectedSourceVisible": false, + "privateContextSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "walletPrivateMaterialVisible": false, + "settlementPrivatePayloadVisible": false, + "forbiddenPayloadClasses": [ + "secret-values", + "provider-tokens", + "wallet-private-material", + "settlement-private-payloads", + "protected-source-payloads", + "raw-protected-prompts", + "raw-prompt-text", + "raw-interpolated-prompts", + "raw-provider-responses", + "private-context", + "unpaid-assetpack-source" + ], + "sourceStats": { + "sourceRootCount": 8, + "sourceRootPresentCount": 8, + "sourceFileCount": 8, + "sourceFileHashes": { + ".bitcode/v41-readfitsfinding-prompt-hardening.json": "2b19151059efe156c6e135ac", + ".bitcode/v41-reading-prompt-benchmark-baselines.json": "ee3f4a7261aeea5da1ccb95c", + "packages/pipelines/asset-pack/src/depository-search.ts": "e983ef9a11ae67d867ebaa99", + "packages/pipelines/asset-pack/src/embedding-config.ts": "e1dc97ae0aafc3cf8464ba4a", + "packages/pipelines/asset-pack/src/read-fits-finding-runtime.ts": "4e89eacdc2b73bd6bc70a085", + "packages/protocol/src/canonical/read-fits-finding-search-embeddings.js": "fcc347d81355cc1168d01247", + "packages/protocol/src/canonical/v41-readfitsfinding-prompt-hardening.js": "43ca6f431fe4e21d34c58ae0", + "packages/protocol/src/canonical/v41-reading-prompt-benchmark-baselines.js": "983d97655980b9a56e1481f0" + } + }, + "requiredPredicateCount": 5, + "passedPredicateCount": 5, + "failedPredicateIds": [], + "passed": true + }, + { + "reportRowId": "conversation-interface-post-rewrite-benchmark-delta", + "label": "Conversation, tool-definition, MCP API, ChatGPT App, and Terminal prompt rewrites bind to benchmark metadata", + "sourceRoots": [ + "packages/protocol/src/canonical/v41-conversation-tool-interface-prompt-rewrite.js", + ".bitcode/v41-conversation-tool-interface-prompt-rewrite.json", + "packages/api/src/conversations/stream-events.ts", + "packages/api/src/conversations/telemetry.ts", + "uapi/components/base/bitcode/execution/pipeline-execution-log.tsx", + "uapi/components/base/bitcode/execution/pipeline-execution-log-header.tsx" + ], + "promptProgramArtifactIds": [ + "v41-conversation-tool-interface-prompt-rewrite" + ], + "metricIds": [ + "benchmark_fixture_result_projection", + "rich_stream_prompt_telemetry_projection", + "source_safe_benchmark_report_disclosure" + ], + "telemetryReceiptIds": [ + "ConversationStreamEvent", + "ConversationTelemetryProofHook", + "PipelineExecutionLog" + ], + "requiredPredicateIds": [ + "gate7-conversation-artifact-passing", + "conversation-stream-template-id-only", + "conversation-telemetry-redacts-protected-context", + "pipeline-log-projects-prompt-result-disclosure", + "conversation-interface-artifact-source-safe" + ], + "rowRoot": "v41-prompt-program-benchmark-report-row:ce6c78c44fdd870b2c650253", + "benchmarkReportMode": "post-rewrite-source-safe-metadata", + "sourceSafetyClass": "source_safe_prompt_program_benchmark_telemetry_metadata", + "sourceSafeMetadataOnly": true, + "disclosureTiers": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "prompt_registry_lineage_source_safe", + "benchmark_fixture_id_source_safe", + "benchmark_delta_score_source_safe", + "telemetry_event_identity_source_safe", + "failsafe_receipt_identity_source_safe", + "thricified_generation_identity_source_safe", + "parsed_output_shape_source_safe", + "schema_verdict_source_safe", + "repair_hook_identity_source_safe", + "source_hash_source_safe", + "raw_prompt_text_private", + "raw_interpolated_prompt_private", + "raw_provider_response_private", + "protected_prompt_private", + "protected_source_private", + "private_context_private", + "unpaid_assetpack_source_private", + "wallet_private_material_private", + "settlement_private_payload_private" + ], + "rawPromptTextSerialized": false, + "rawInterpolatedPromptSerialized": false, + "rawProviderResponseSerialized": false, + "protectedPromptSerialized": false, + "protectedSourceVisible": false, + "privateContextSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "walletPrivateMaterialVisible": false, + "settlementPrivatePayloadVisible": false, + "forbiddenPayloadClasses": [ + "secret-values", + "provider-tokens", + "wallet-private-material", + "settlement-private-payloads", + "protected-source-payloads", + "raw-protected-prompts", + "raw-prompt-text", + "raw-interpolated-prompts", + "raw-provider-responses", + "private-context", + "unpaid-assetpack-source" + ], + "sourceStats": { + "sourceRootCount": 6, + "sourceRootPresentCount": 6, + "sourceFileCount": 6, + "sourceFileHashes": { + ".bitcode/v41-conversation-tool-interface-prompt-rewrite.json": "5d4736a363f57382f62d6a3d", + "packages/api/src/conversations/stream-events.ts": "049a26286565e121f20c6874", + "packages/api/src/conversations/telemetry.ts": "9ebc4836fe0e2cde2699cb7e", + "packages/protocol/src/canonical/v41-conversation-tool-interface-prompt-rewrite.js": "3bf1de40f2dfd8f0d4e9be76", + "uapi/components/base/bitcode/execution/pipeline-execution-log-header.tsx": "7ba7c3135c5e04d1eb1a7771", + "uapi/components/base/bitcode/execution/pipeline-execution-log.tsx": "bb7e1773303e04b9027ae2c7" + } + }, + "requiredPredicateCount": 5, + "passedPredicateCount": 5, + "failedPredicateIds": [], + "passed": true + }, + { + "reportRowId": "registry-lineage-version-telemetry-binding", + "label": "Prompt registry lineage and version metadata are projected as ids, roots, counts, and schema names", + "sourceRoots": [ + "packages/protocol/src/canonical/v41-registry-interpolation-contracts.js", + ".bitcode/v41-registry-interpolation-contracts.json", + "packages/prompts/src/benchmarking/types.ts", + "packages/prompts/src/benchmarking/runner.ts", + "packages/pipelines/asset-pack/src/reading-pipeline-contract.ts" + ], + "promptProgramArtifactIds": [ + "v41-registry-interpolation-contracts", + "v38-prompt-benchmark-report" + ], + "metricIds": [ + "prompt_lineage_registry_versioning", + "post_rewrite_promptpart_delta", + "post_rewrite_prompt_delta" + ], + "telemetryReceiptIds": [ + "PromptPartBenchmarkResult", + "PromptBenchmarkResult", + "RegistryInterpolationReceipt" + ], + "requiredPredicateIds": [ + "gate3-registry-contracts-passing", + "benchmark-types-have-pbversion-and-quality-gates", + "benchmark-runner-handles-promptpart-and-prompt", + "registry-contracts-bind-hierarchy", + "registry-lineage-source-safe" + ], + "rowRoot": "v41-prompt-program-benchmark-report-row:97cf2111c50bbdcd4ea18e1c", + "benchmarkReportMode": "post-rewrite-source-safe-metadata", + "sourceSafetyClass": "source_safe_prompt_program_benchmark_telemetry_metadata", + "sourceSafeMetadataOnly": true, + "disclosureTiers": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "prompt_registry_lineage_source_safe", + "benchmark_fixture_id_source_safe", + "benchmark_delta_score_source_safe", + "telemetry_event_identity_source_safe", + "failsafe_receipt_identity_source_safe", + "thricified_generation_identity_source_safe", + "parsed_output_shape_source_safe", + "schema_verdict_source_safe", + "repair_hook_identity_source_safe", + "source_hash_source_safe", + "raw_prompt_text_private", + "raw_interpolated_prompt_private", + "raw_provider_response_private", + "protected_prompt_private", + "protected_source_private", + "private_context_private", + "unpaid_assetpack_source_private", + "wallet_private_material_private", + "settlement_private_payload_private" + ], + "rawPromptTextSerialized": false, + "rawInterpolatedPromptSerialized": false, + "rawProviderResponseSerialized": false, + "protectedPromptSerialized": false, + "protectedSourceVisible": false, + "privateContextSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "walletPrivateMaterialVisible": false, + "settlementPrivatePayloadVisible": false, + "forbiddenPayloadClasses": [ + "secret-values", + "provider-tokens", + "wallet-private-material", + "settlement-private-payloads", + "protected-source-payloads", + "raw-protected-prompts", + "raw-prompt-text", + "raw-interpolated-prompts", + "raw-provider-responses", + "private-context", + "unpaid-assetpack-source" + ], + "sourceStats": { + "sourceRootCount": 5, + "sourceRootPresentCount": 5, + "sourceFileCount": 5, + "sourceFileHashes": { + ".bitcode/v41-registry-interpolation-contracts.json": "ee84aff9248f77fc811b2a5a", + "packages/pipelines/asset-pack/src/reading-pipeline-contract.ts": "b19eba649308640b1854dea6", + "packages/prompts/src/benchmarking/runner.ts": "dfe65501927783fca7cd3305", + "packages/prompts/src/benchmarking/types.ts": "e996cfbbcf4906498f2821b0", + "packages/protocol/src/canonical/v41-registry-interpolation-contracts.js": "b2853d91a04f7dd4b40d77fc" + } + }, + "requiredPredicateCount": 5, + "passedPredicateCount": 5, + "failedPredicateIds": [], + "passed": true + }, + { + "reportRowId": "failsafe-thricified-inference-receipt-projection", + "label": "PTRR steps, Failsafe sequences, ThricifiedGeneration calls, raw-output roots, and parsed outputs remain traceable", + "sourceRoots": [ + "packages/protocol/src/canonical/ptrr-failsafe-thricified-stack.js", + "packages/protocol/src/canonical/inference-telemetry-disclosure-report.js", + ".bitcode/v38-disclosure-boundary-report.json", + "packages/pipelines/asset-pack/src/bounded-structured-inference.ts", + "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts" + ], + "promptProgramArtifactIds": [ + "v38-ptrr-failsafe-thricified-stack", + "v38-inference-telemetry-disclosure-report" + ], + "metricIds": [ + "failsafe_thricified_receipt_projection", + "parsed_output_schema_verdict_projection", + "rich_stream_prompt_telemetry_projection" + ], + "telemetryReceiptIds": [ + "pipeline_phase", + "agent", + "ptrr_step", + "failsafe", + "thricified_generation", + "interpolated_prompt", + "raw_response", + "parsed_output", + "schema_verdict" + ], + "requiredPredicateIds": [ + "v38-ptrr-stack-passing", + "v38-inference-telemetry-passing", + "inference-telemetry-has-required-levels", + "bounded-inference-records-raw-and-parsed", + "observability-projects-generation-receipts", + "failsafe-thricified-source-safe" + ], + "rowRoot": "v41-prompt-program-benchmark-report-row:6fcc617d04f442627fcf7a7a", + "benchmarkReportMode": "post-rewrite-source-safe-metadata", + "sourceSafetyClass": "source_safe_prompt_program_benchmark_telemetry_metadata", + "sourceSafeMetadataOnly": true, + "disclosureTiers": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "prompt_registry_lineage_source_safe", + "benchmark_fixture_id_source_safe", + "benchmark_delta_score_source_safe", + "telemetry_event_identity_source_safe", + "failsafe_receipt_identity_source_safe", + "thricified_generation_identity_source_safe", + "parsed_output_shape_source_safe", + "schema_verdict_source_safe", + "repair_hook_identity_source_safe", + "source_hash_source_safe", + "raw_prompt_text_private", + "raw_interpolated_prompt_private", + "raw_provider_response_private", + "protected_prompt_private", + "protected_source_private", + "private_context_private", + "unpaid_assetpack_source_private", + "wallet_private_material_private", + "settlement_private_payload_private" + ], + "rawPromptTextSerialized": false, + "rawInterpolatedPromptSerialized": false, + "rawProviderResponseSerialized": false, + "protectedPromptSerialized": false, + "protectedSourceVisible": false, + "privateContextSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "walletPrivateMaterialVisible": false, + "settlementPrivatePayloadVisible": false, + "forbiddenPayloadClasses": [ + "secret-values", + "provider-tokens", + "wallet-private-material", + "settlement-private-payloads", + "protected-source-payloads", + "raw-protected-prompts", + "raw-prompt-text", + "raw-interpolated-prompts", + "raw-provider-responses", + "private-context", + "unpaid-assetpack-source" + ], + "sourceStats": { + "sourceRootCount": 5, + "sourceRootPresentCount": 5, + "sourceFileCount": 5, + "sourceFileHashes": { + ".bitcode/v38-disclosure-boundary-report.json": "d71a4dda219c03d5fc031ee2", + "packages/pipelines/asset-pack/src/bounded-structured-inference.ts": "c918c2d3ef7da0ab94a2a7ed", + "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts": "3cc1dbd2a18b7db46440d0ba", + "packages/protocol/src/canonical/inference-telemetry-disclosure-report.js": "fe0fac851d12b6719fbefd22", + "packages/protocol/src/canonical/ptrr-failsafe-thricified-stack.js": "535a9e2a4e4f2fc9fa8a808c" + } + }, + "requiredPredicateCount": 6, + "passedPredicateCount": 6, + "failedPredicateIds": [], + "passed": true + }, + { + "reportRowId": "rich-stream-source-safe-telemetry-projection", + "label": "Rich execution-log streams expose prompt and result posture as source-safe metadata for live operator inspection", + "sourceRoots": [ + "packages/api/src/conversations/stream-events.ts", + "packages/api/src/conversations/telemetry.ts", + "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts", + "packages/pipelines/asset-pack/src/reading-operational-telemetry-repair-readback.ts", + "uapi/components/base/bitcode/execution/pipeline-execution-log.tsx", + "uapi/components/base/bitcode/execution/pipeline-execution-log-header.tsx", + "uapi/tests/conversationStreamPipelineLog.test.tsx", + "uapi/tests/pipelineExecutionLogHeader.test.tsx" + ], + "promptProgramArtifactIds": [ + "v38-inference-telemetry-disclosure-report", + "v39-operational-telemetry-repair-readback" + ], + "metricIds": [ + "rich_stream_prompt_telemetry_projection", + "repair_hook_redaction_posture", + "source_safe_benchmark_report_disclosure" + ], + "telemetryReceiptIds": [ + "ReadingPipelineTelemetryProjection", + "ReadingOperationalTelemetryEvent", + "ConversationStreamEvent" + ], + "requiredPredicateIds": [ + "reading-operational-telemetry-passing", + "stream-events-sanitize-metadata", + "rich-log-header-renders-prompt-result-posture", + "rich-log-tests-cover-telemetry-disclosure", + "rich-stream-source-safe-only" + ], + "rowRoot": "v41-prompt-program-benchmark-report-row:9e23ac95e57bad9e0d736a9a", + "benchmarkReportMode": "post-rewrite-source-safe-metadata", + "sourceSafetyClass": "source_safe_prompt_program_benchmark_telemetry_metadata", + "sourceSafeMetadataOnly": true, + "disclosureTiers": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "prompt_registry_lineage_source_safe", + "benchmark_fixture_id_source_safe", + "benchmark_delta_score_source_safe", + "telemetry_event_identity_source_safe", + "failsafe_receipt_identity_source_safe", + "thricified_generation_identity_source_safe", + "parsed_output_shape_source_safe", + "schema_verdict_source_safe", + "repair_hook_identity_source_safe", + "source_hash_source_safe", + "raw_prompt_text_private", + "raw_interpolated_prompt_private", + "raw_provider_response_private", + "protected_prompt_private", + "protected_source_private", + "private_context_private", + "unpaid_assetpack_source_private", + "wallet_private_material_private", + "settlement_private_payload_private" + ], + "rawPromptTextSerialized": false, + "rawInterpolatedPromptSerialized": false, + "rawProviderResponseSerialized": false, + "protectedPromptSerialized": false, + "protectedSourceVisible": false, + "privateContextSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "walletPrivateMaterialVisible": false, + "settlementPrivatePayloadVisible": false, + "forbiddenPayloadClasses": [ + "secret-values", + "provider-tokens", + "wallet-private-material", + "settlement-private-payloads", + "protected-source-payloads", + "raw-protected-prompts", + "raw-prompt-text", + "raw-interpolated-prompts", + "raw-provider-responses", + "private-context", + "unpaid-assetpack-source" + ], + "sourceStats": { + "sourceRootCount": 8, + "sourceRootPresentCount": 8, + "sourceFileCount": 8, + "sourceFileHashes": { + "packages/api/src/conversations/stream-events.ts": "049a26286565e121f20c6874", + "packages/api/src/conversations/telemetry.ts": "9ebc4836fe0e2cde2699cb7e", + "packages/pipelines/asset-pack/src/reading-operational-telemetry-repair-readback.ts": "4b0d32180a32720cb0086ffe", + "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts": "3cc1dbd2a18b7db46440d0ba", + "uapi/components/base/bitcode/execution/pipeline-execution-log-header.tsx": "7ba7c3135c5e04d1eb1a7771", + "uapi/components/base/bitcode/execution/pipeline-execution-log.tsx": "bb7e1773303e04b9027ae2c7", + "uapi/tests/conversationStreamPipelineLog.test.tsx": "73e3ba9c6f058ffe4418b3e4", + "uapi/tests/pipelineExecutionLogHeader.test.tsx": "4835ee31e6397899e58b4183" + } + }, + "requiredPredicateCount": 5, + "passedPredicateCount": 5, + "failedPredicateIds": [], + "passed": true + }, + { + "reportRowId": "repair-hooks-parsed-output-redaction-posture", + "label": "Repair hooks and parsed output projections remain debuggable without leaking source, prompts, responses, or settlement payloads", + "sourceRoots": [ + "packages/pipelines/asset-pack/src/reading-operational-telemetry-repair-readback.ts", + "packages/protocol/src/canonical/v39-operational-telemetry-repair-readback.js", + ".bitcode/v39-operational-telemetry-repair-readback.json", + "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts", + "packages/api/src/conversations/telemetry.ts" + ], + "promptProgramArtifactIds": [ + "v39-operational-telemetry-repair-readback", + "v38-inference-telemetry-disclosure-report" + ], + "metricIds": [ + "repair_hook_redaction_posture", + "parsed_output_schema_verdict_projection", + "source_safe_benchmark_report_disclosure" + ], + "telemetryReceiptIds": [ + "repair", + "schema_verdict", + "proof_roots_ids_counts_states_and_redacted_error_classes_only" + ], + "requiredPredicateIds": [ + "repair-hooks-present", + "repair-readback-redacts-raw-prompts", + "conversation-telemetry-redacts-secret-values", + "parsed-output-shape-source-safe", + "repair-hook-artifacts-source-safe" + ], + "rowRoot": "v41-prompt-program-benchmark-report-row:ccc74c7f242dcecfa971d66f", + "benchmarkReportMode": "post-rewrite-source-safe-metadata", + "sourceSafetyClass": "source_safe_prompt_program_benchmark_telemetry_metadata", + "sourceSafeMetadataOnly": true, + "disclosureTiers": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "prompt_registry_lineage_source_safe", + "benchmark_fixture_id_source_safe", + "benchmark_delta_score_source_safe", + "telemetry_event_identity_source_safe", + "failsafe_receipt_identity_source_safe", + "thricified_generation_identity_source_safe", + "parsed_output_shape_source_safe", + "schema_verdict_source_safe", + "repair_hook_identity_source_safe", + "source_hash_source_safe", + "raw_prompt_text_private", + "raw_interpolated_prompt_private", + "raw_provider_response_private", + "protected_prompt_private", + "protected_source_private", + "private_context_private", + "unpaid_assetpack_source_private", + "wallet_private_material_private", + "settlement_private_payload_private" + ], + "rawPromptTextSerialized": false, + "rawInterpolatedPromptSerialized": false, + "rawProviderResponseSerialized": false, + "protectedPromptSerialized": false, + "protectedSourceVisible": false, + "privateContextSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "walletPrivateMaterialVisible": false, + "settlementPrivatePayloadVisible": false, + "forbiddenPayloadClasses": [ + "secret-values", + "provider-tokens", + "wallet-private-material", + "settlement-private-payloads", + "protected-source-payloads", + "raw-protected-prompts", + "raw-prompt-text", + "raw-interpolated-prompts", + "raw-provider-responses", + "private-context", + "unpaid-assetpack-source" + ], + "sourceStats": { + "sourceRootCount": 5, + "sourceRootPresentCount": 5, + "sourceFileCount": 5, + "sourceFileHashes": { + ".bitcode/v39-operational-telemetry-repair-readback.json": "7522f7709b545d11b7f9ba74", + "packages/api/src/conversations/telemetry.ts": "9ebc4836fe0e2cde2699cb7e", + "packages/pipelines/asset-pack/src/reading-operational-telemetry-repair-readback.ts": "4b0d32180a32720cb0086ffe", + "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts": "3cc1dbd2a18b7db46440d0ba", + "packages/protocol/src/canonical/v39-operational-telemetry-repair-readback.js": "1647b3a170af58333c1553a3" + } + }, + "requiredPredicateCount": 5, + "passedPredicateCount": 5, + "failedPredicateIds": [], + "passed": true + }, + { + "reportRowId": "gate8-tests-docs-workflows", + "label": "Gate 8 tests, scripts, docs, generated artifact, package exports, and workflows close benchmark telemetry integration", + "sourceRoots": [ + "packages/protocol/src/canonical/v41-prompt-program-benchmark-report.js", + "packages/protocol/test/v41-prompt-program-benchmark-report.test.js", + "scripts/generate-v41-prompt-program-benchmark-report.mjs", + "scripts/check-v41-gate8-prompt-program-benchmark-report.mjs", + "BITCODE_SPEC_V41.md", + "BITCODE_SPEC_V41_DELTA.md", + "BITCODE_SPEC_V41_NOTES.md", + "BITCODE_SPEC_V41_PARITY_MATRIX.md", + "SPECIFICATIONS_ROADMAP.md", + "README.md", + "packages/protocol/README.md", + "package.json", + ".github/workflows/bitcode-gate-quality.yml", + ".github/workflows/bitcode-canon-quality.yml" + ], + "promptProgramArtifactIds": [ + "v41-prompt-program-benchmark-report" + ], + "metricIds": [ + "source_safe_benchmark_report_disclosure", + "repair_hook_redaction_posture" + ], + "telemetryReceiptIds": [ + "V41PromptProgramBenchmarkReportArtifact" + ], + "requiredPredicateIds": [ + "gate8-source-exists", + "gate8-test-exists", + "gate8-generator-exists", + "gate8-checker-exists", + "package-json-exposes-gate8-scripts", + "workflows-run-gate8-checker", + "spec-documents-gate8-closure", + "roadmap-documents-gate8-closure", + "readmes-document-gate8-helpers" + ], + "rowRoot": "v41-prompt-program-benchmark-report-row:695079e05da055000332fa65", + "benchmarkReportMode": "post-rewrite-source-safe-metadata", + "sourceSafetyClass": "source_safe_prompt_program_benchmark_telemetry_metadata", + "sourceSafeMetadataOnly": true, + "disclosureTiers": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "prompt_registry_lineage_source_safe", + "benchmark_fixture_id_source_safe", + "benchmark_delta_score_source_safe", + "telemetry_event_identity_source_safe", + "failsafe_receipt_identity_source_safe", + "thricified_generation_identity_source_safe", + "parsed_output_shape_source_safe", + "schema_verdict_source_safe", + "repair_hook_identity_source_safe", + "source_hash_source_safe", + "raw_prompt_text_private", + "raw_interpolated_prompt_private", + "raw_provider_response_private", + "protected_prompt_private", + "protected_source_private", + "private_context_private", + "unpaid_assetpack_source_private", + "wallet_private_material_private", + "settlement_private_payload_private" + ], + "rawPromptTextSerialized": false, + "rawInterpolatedPromptSerialized": false, + "rawProviderResponseSerialized": false, + "protectedPromptSerialized": false, + "protectedSourceVisible": false, + "privateContextSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "walletPrivateMaterialVisible": false, + "settlementPrivatePayloadVisible": false, + "forbiddenPayloadClasses": [ + "secret-values", + "provider-tokens", + "wallet-private-material", + "settlement-private-payloads", + "protected-source-payloads", + "raw-protected-prompts", + "raw-prompt-text", + "raw-interpolated-prompts", + "raw-provider-responses", + "private-context", + "unpaid-assetpack-source" + ], + "sourceStats": { + "sourceRootCount": 14, + "sourceRootPresentCount": 14, + "sourceFileCount": 14, + "sourceFileHashes": { + ".github/workflows/bitcode-canon-quality.yml": "75c13714eda5b1bf46ad896e", + ".github/workflows/bitcode-gate-quality.yml": "4498ef474a10506a6cf781ce", + "BITCODE_SPEC_V41.md": "35ed9e60cecdcb8bd820caee", + "BITCODE_SPEC_V41_DELTA.md": "68092aae6b6f8916ee5652a0", + "BITCODE_SPEC_V41_NOTES.md": "7ed03aa76aa39be60d45ab0c", + "BITCODE_SPEC_V41_PARITY_MATRIX.md": "77e1bfa9c3dd2d549d5123e3", + "README.md": "64b27506f4fb734632abbb4f", + "SPECIFICATIONS_ROADMAP.md": "ea44c2816a4548954f32a742", + "package.json": "319d175fa7fedbc48b90cb37", + "packages/protocol/README.md": "b755f91f456d78b111c14794", + "packages/protocol/src/canonical/v41-prompt-program-benchmark-report.js": "8f576df874b9e5d8fc71c2bd", + "packages/protocol/test/v41-prompt-program-benchmark-report.test.js": "d0d2609b55c61933ca16a63a", + "scripts/check-v41-gate8-prompt-program-benchmark-report.mjs": "f11d962a0fcd80785ee0a6cc", + "scripts/generate-v41-prompt-program-benchmark-report.mjs": "d3c64e23a5ce6cda60bec813" + } + }, + "requiredPredicateCount": 9, + "passedPredicateCount": 9, + "failedPredicateIds": [], + "passed": true + } + ], + "sourceEvidence": [ + { + "sourcePath": ".bitcode/v38-disclosure-boundary-report.json", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "b6fb979d592616ff665d5f58" + }, + { + "sourcePath": ".bitcode/v39-operational-telemetry-repair-readback.json", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "61891cc08936441e0de04e9a" + }, + { + "sourcePath": ".bitcode/v41-conversation-tool-interface-prompt-rewrite.json", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "2f95c74c96e01d605c9b50f3" + }, + { + "sourcePath": ".bitcode/v41-promptpart-prompt-inventory.json", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "e00ab96a2e02f039f623470c" + }, + { + "sourcePath": ".bitcode/v41-readfitsfinding-prompt-hardening.json", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "cf40b3133a2bd780989a0cc2" + }, + { + "sourcePath": ".bitcode/v41-reading-prompt-benchmark-baselines.json", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "d1b7f3a864b2e9080b587a4e" + }, + { + "sourcePath": ".bitcode/v41-readneed-prompt-hardening.json", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "a0febbc3fafec60232baadd2" + }, + { + "sourcePath": ".bitcode/v41-registry-interpolation-contracts.json", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "17c9e8f9dd67fc19d9f383fc" + }, + { + "sourcePath": ".github/workflows/bitcode-canon-quality.yml", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "16eade0a1f25db9e1cdc1302" + }, + { + "sourcePath": ".github/workflows/bitcode-gate-quality.yml", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "e569c70fad82f54b7a7b3fb7" + }, + { + "sourcePath": "BITCODE_SPEC_V41.md", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "e05b79444b6fa82d9be72048" + }, + { + "sourcePath": "BITCODE_SPEC_V41_DELTA.md", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "41df55b920117a59a9df97eb" + }, + { + "sourcePath": "BITCODE_SPEC_V41_NOTES.md", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "2fe5e35a718e1a85db7d2f8e" + }, + { + "sourcePath": "BITCODE_SPEC_V41_PARITY_MATRIX.md", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "efb5a164364287bcded23143" + }, + { + "sourcePath": "README.md", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "8f5dadb3c3fa77260cd12675" + }, + { + "sourcePath": "SPECIFICATIONS_ROADMAP.md", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "eb9a77f8f7010d3caacfa377" + }, + { + "sourcePath": "package.json", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "4f0e01b9077d7719cd501ddc" + }, + { + "sourcePath": "packages/api/src/conversations/stream-events.ts", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "2d4b1bbfab7f848c59fae8d8" + }, + { + "sourcePath": "packages/api/src/conversations/telemetry.ts", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "3dd00b34096a0084485bf679" + }, + { + "sourcePath": "packages/pipelines/asset-pack/src/bounded-structured-inference.ts", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "0e8928a96135f4a373927844" + }, + { + "sourcePath": "packages/pipelines/asset-pack/src/depository-search.ts", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "203e1fad5fd26e518421c024" + }, + { + "sourcePath": "packages/pipelines/asset-pack/src/embedding-config.ts", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "15469b3c0f1714d73ec3c669" + }, + { + "sourcePath": "packages/pipelines/asset-pack/src/read-fits-finding-runtime.ts", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "5dd5c7e3e2fbfee87dcf5560" + }, + { + "sourcePath": "packages/pipelines/asset-pack/src/reading-operational-telemetry-repair-readback.ts", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "94220ec79a066d950348f74d" + }, + { + "sourcePath": "packages/pipelines/asset-pack/src/reading-pipeline-contract.ts", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "4bf982618ce45ebc06343c81" + }, + { + "sourcePath": "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "f456521f2ce76cc8000539fd" + }, + { + "sourcePath": "packages/prompts/src/benchmarking/runner.ts", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "066c62096a75d06fe891c8e6" + }, + { + "sourcePath": "packages/prompts/src/benchmarking/types.ts", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "c2abeb7018d71d242d41c87a" + }, + { + "sourcePath": "packages/protocol/README.md", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "46f39a83141a8d2bdf41cf37" + }, + { + "sourcePath": "packages/protocol/src/canonical/inference-telemetry-disclosure-report.js", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "3694cceef038b5b672b30794" + }, + { + "sourcePath": "packages/protocol/src/canonical/ptrr-failsafe-thricified-stack.js", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "a5a0a0cbd1779ab1fe13e304" + }, + { + "sourcePath": "packages/protocol/src/canonical/read-fits-finding-search-embeddings.js", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "01ef8feac03caea21f3b9df3" + }, + { + "sourcePath": "packages/protocol/src/canonical/v39-operational-telemetry-repair-readback.js", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "0269806cf9b3ddb0454aca93" + }, + { + "sourcePath": "packages/protocol/src/canonical/v41-conversation-tool-interface-prompt-rewrite.js", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "05799d405f416910426e7072" + }, + { + "sourcePath": "packages/protocol/src/canonical/v41-prompt-program-benchmark-report.js", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "4ab06ab3eefe30d40a23a0cc" + }, + { + "sourcePath": "packages/protocol/src/canonical/v41-promptpart-prompt-inventory.js", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "a84b7d0b1f4783b727467564" + }, + { + "sourcePath": "packages/protocol/src/canonical/v41-readfitsfinding-prompt-hardening.js", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "b41f96c1bb11919ad0ac73cc" + }, + { + "sourcePath": "packages/protocol/src/canonical/v41-reading-prompt-benchmark-baselines.js", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "2c1ed8985639259fe13d0cf0" + }, + { + "sourcePath": "packages/protocol/src/canonical/v41-readneed-prompt-hardening.js", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "e85e22a201676edf7454d77f" + }, + { + "sourcePath": "packages/protocol/src/canonical/v41-registry-interpolation-contracts.js", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "680b3458ac8e92a42d8ae55e" + }, + { + "sourcePath": "packages/protocol/test/v41-prompt-program-benchmark-report.test.js", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "17e088da1ae39dc757a492e9" + }, + { + "sourcePath": "scripts/check-v41-gate8-prompt-program-benchmark-report.mjs", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "ca8470c018a07a7a167532ec" + }, + { + "sourcePath": "scripts/generate-v41-prompt-program-benchmark-report.mjs", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "7cf03ec77f881f1e99ca81a8" + }, + { + "sourcePath": "uapi/components/base/bitcode/execution/pipeline-execution-log-header.tsx", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "f9c3d094eb331f25026cc948" + }, + { + "sourcePath": "uapi/components/base/bitcode/execution/pipeline-execution-log.tsx", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "9e3f435fead756734ba9175c" + }, + { + "sourcePath": "uapi/tests/conversationStreamPipelineLog.test.tsx", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "fc41f8ce1326ccfa5357e0ce" + }, + { + "sourcePath": "uapi/tests/pipelineExecutionLogHeader.test.tsx", + "exists": true, + "legacy": false, + "fileCount": 1, + "sourceRootHash": "d67d77d055943655f2ec4e55" + } + ], + "sourceStatsByRow": { + "post-rewrite-prompt-inventory-delta": { + "sourceRootCount": 5, + "sourceRootPresentCount": 5, + "sourceFileCount": 5, + "sourceFileHashes": { + ".bitcode/v41-conversation-tool-interface-prompt-rewrite.json": "5d4736a363f57382f62d6a3d", + ".bitcode/v41-promptpart-prompt-inventory.json": "927738d4b3bdab4f2f9734e7", + ".bitcode/v41-readfitsfinding-prompt-hardening.json": "2b19151059efe156c6e135ac", + ".bitcode/v41-readneed-prompt-hardening.json": "148ce8c1c9bc8ebf03f1b2d2", + "packages/protocol/src/canonical/v41-promptpart-prompt-inventory.js": "75952a25cc72ee8b58d5a956" + } + }, + "readneed-post-rewrite-benchmark-delta": { + "sourceRootCount": 6, + "sourceRootPresentCount": 6, + "sourceFileCount": 6, + "sourceFileHashes": { + ".bitcode/v41-reading-prompt-benchmark-baselines.json": "ee3f4a7261aeea5da1ccb95c", + ".bitcode/v41-readneed-prompt-hardening.json": "148ce8c1c9bc8ebf03f1b2d2", + "packages/pipelines/asset-pack/src/reading-pipeline-contract.ts": "b19eba649308640b1854dea6", + "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts": "3cc1dbd2a18b7db46440d0ba", + "packages/protocol/src/canonical/v41-reading-prompt-benchmark-baselines.js": "983d97655980b9a56e1481f0", + "packages/protocol/src/canonical/v41-readneed-prompt-hardening.js": "8b1eed4bd12fffb273f18711" + } + }, + "readfitsfinding-post-rewrite-benchmark-delta": { + "sourceRootCount": 8, + "sourceRootPresentCount": 8, + "sourceFileCount": 8, + "sourceFileHashes": { + ".bitcode/v41-readfitsfinding-prompt-hardening.json": "2b19151059efe156c6e135ac", + ".bitcode/v41-reading-prompt-benchmark-baselines.json": "ee3f4a7261aeea5da1ccb95c", + "packages/pipelines/asset-pack/src/depository-search.ts": "e983ef9a11ae67d867ebaa99", + "packages/pipelines/asset-pack/src/embedding-config.ts": "e1dc97ae0aafc3cf8464ba4a", + "packages/pipelines/asset-pack/src/read-fits-finding-runtime.ts": "4e89eacdc2b73bd6bc70a085", + "packages/protocol/src/canonical/read-fits-finding-search-embeddings.js": "fcc347d81355cc1168d01247", + "packages/protocol/src/canonical/v41-readfitsfinding-prompt-hardening.js": "43ca6f431fe4e21d34c58ae0", + "packages/protocol/src/canonical/v41-reading-prompt-benchmark-baselines.js": "983d97655980b9a56e1481f0" + } + }, + "conversation-interface-post-rewrite-benchmark-delta": { + "sourceRootCount": 6, + "sourceRootPresentCount": 6, + "sourceFileCount": 6, + "sourceFileHashes": { + ".bitcode/v41-conversation-tool-interface-prompt-rewrite.json": "5d4736a363f57382f62d6a3d", + "packages/api/src/conversations/stream-events.ts": "049a26286565e121f20c6874", + "packages/api/src/conversations/telemetry.ts": "9ebc4836fe0e2cde2699cb7e", + "packages/protocol/src/canonical/v41-conversation-tool-interface-prompt-rewrite.js": "3bf1de40f2dfd8f0d4e9be76", + "uapi/components/base/bitcode/execution/pipeline-execution-log-header.tsx": "7ba7c3135c5e04d1eb1a7771", + "uapi/components/base/bitcode/execution/pipeline-execution-log.tsx": "bb7e1773303e04b9027ae2c7" + } + }, + "registry-lineage-version-telemetry-binding": { + "sourceRootCount": 5, + "sourceRootPresentCount": 5, + "sourceFileCount": 5, + "sourceFileHashes": { + ".bitcode/v41-registry-interpolation-contracts.json": "ee84aff9248f77fc811b2a5a", + "packages/pipelines/asset-pack/src/reading-pipeline-contract.ts": "b19eba649308640b1854dea6", + "packages/prompts/src/benchmarking/runner.ts": "dfe65501927783fca7cd3305", + "packages/prompts/src/benchmarking/types.ts": "e996cfbbcf4906498f2821b0", + "packages/protocol/src/canonical/v41-registry-interpolation-contracts.js": "b2853d91a04f7dd4b40d77fc" + } + }, + "failsafe-thricified-inference-receipt-projection": { + "sourceRootCount": 5, + "sourceRootPresentCount": 5, + "sourceFileCount": 5, + "sourceFileHashes": { + ".bitcode/v38-disclosure-boundary-report.json": "d71a4dda219c03d5fc031ee2", + "packages/pipelines/asset-pack/src/bounded-structured-inference.ts": "c918c2d3ef7da0ab94a2a7ed", + "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts": "3cc1dbd2a18b7db46440d0ba", + "packages/protocol/src/canonical/inference-telemetry-disclosure-report.js": "fe0fac851d12b6719fbefd22", + "packages/protocol/src/canonical/ptrr-failsafe-thricified-stack.js": "535a9e2a4e4f2fc9fa8a808c" + } + }, + "rich-stream-source-safe-telemetry-projection": { + "sourceRootCount": 8, + "sourceRootPresentCount": 8, + "sourceFileCount": 8, + "sourceFileHashes": { + "packages/api/src/conversations/stream-events.ts": "049a26286565e121f20c6874", + "packages/api/src/conversations/telemetry.ts": "9ebc4836fe0e2cde2699cb7e", + "packages/pipelines/asset-pack/src/reading-operational-telemetry-repair-readback.ts": "4b0d32180a32720cb0086ffe", + "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts": "3cc1dbd2a18b7db46440d0ba", + "uapi/components/base/bitcode/execution/pipeline-execution-log-header.tsx": "7ba7c3135c5e04d1eb1a7771", + "uapi/components/base/bitcode/execution/pipeline-execution-log.tsx": "bb7e1773303e04b9027ae2c7", + "uapi/tests/conversationStreamPipelineLog.test.tsx": "73e3ba9c6f058ffe4418b3e4", + "uapi/tests/pipelineExecutionLogHeader.test.tsx": "4835ee31e6397899e58b4183" + } + }, + "repair-hooks-parsed-output-redaction-posture": { + "sourceRootCount": 5, + "sourceRootPresentCount": 5, + "sourceFileCount": 5, + "sourceFileHashes": { + ".bitcode/v39-operational-telemetry-repair-readback.json": "7522f7709b545d11b7f9ba74", + "packages/api/src/conversations/telemetry.ts": "9ebc4836fe0e2cde2699cb7e", + "packages/pipelines/asset-pack/src/reading-operational-telemetry-repair-readback.ts": "4b0d32180a32720cb0086ffe", + "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts": "3cc1dbd2a18b7db46440d0ba", + "packages/protocol/src/canonical/v39-operational-telemetry-repair-readback.js": "1647b3a170af58333c1553a3" + } + }, + "gate8-tests-docs-workflows": { + "sourceRootCount": 14, + "sourceRootPresentCount": 14, + "sourceFileCount": 14, + "sourceFileHashes": { + ".github/workflows/bitcode-canon-quality.yml": "75c13714eda5b1bf46ad896e", + ".github/workflows/bitcode-gate-quality.yml": "4498ef474a10506a6cf781ce", + "BITCODE_SPEC_V41.md": "35ed9e60cecdcb8bd820caee", + "BITCODE_SPEC_V41_DELTA.md": "68092aae6b6f8916ee5652a0", + "BITCODE_SPEC_V41_NOTES.md": "7ed03aa76aa39be60d45ab0c", + "BITCODE_SPEC_V41_PARITY_MATRIX.md": "77e1bfa9c3dd2d549d5123e3", + "README.md": "64b27506f4fb734632abbb4f", + "SPECIFICATIONS_ROADMAP.md": "ea44c2816a4548954f32a742", + "package.json": "319d175fa7fedbc48b90cb37", + "packages/protocol/README.md": "b755f91f456d78b111c14794", + "packages/protocol/src/canonical/v41-prompt-program-benchmark-report.js": "8f576df874b9e5d8fc71c2bd", + "packages/protocol/test/v41-prompt-program-benchmark-report.test.js": "d0d2609b55c61933ca16a63a", + "scripts/check-v41-gate8-prompt-program-benchmark-report.mjs": "f11d962a0fcd80785ee0a6cc", + "scripts/generate-v41-prompt-program-benchmark-report.mjs": "d3c64e23a5ce6cda60bec813" + } + } + }, + "sourcePredicateResults": [ + { + "id": "gate2-inventory-passing", + "rowId": "post-rewrite-prompt-inventory-delta", + "sourcePath": ".bitcode/v41-promptpart-prompt-inventory.json", + "passed": true + }, + { + "id": "gate5-readneed-passing", + "rowId": "post-rewrite-prompt-inventory-delta", + "sourcePath": ".bitcode/v41-readneed-prompt-hardening.json", + "passed": true + }, + { + "id": "gate6-readfitsfinding-passing", + "rowId": "post-rewrite-prompt-inventory-delta", + "sourcePath": ".bitcode/v41-readfitsfinding-prompt-hardening.json", + "passed": true + }, + { + "id": "gate7-conversation-interface-passing", + "rowId": "post-rewrite-prompt-inventory-delta", + "sourcePath": ".bitcode/v41-conversation-tool-interface-prompt-rewrite.json", + "passed": true + }, + { + "id": "post-rewrite-artifacts-present", + "rowId": "post-rewrite-prompt-inventory-delta", + "sourcePath": ".bitcode", + "passed": true + }, + { + "id": "post-rewrite-report-does-not-serialize-raw-prompt-text", + "rowId": "post-rewrite-prompt-inventory-delta", + "sourcePath": "sourceSafety", + "passed": true + }, + { + "id": "gate4-reading-benchmark-passing", + "rowId": "readneed-post-rewrite-benchmark-delta", + "sourcePath": ".bitcode/v41-reading-prompt-benchmark-baselines.json", + "passed": true + }, + { + "id": "readneed-hardening-row-count", + "rowId": "readneed-post-rewrite-benchmark-delta", + "sourcePath": ".bitcode/v41-readneed-prompt-hardening.json", + "passed": true + }, + { + "id": "readneed-benchmark-baselines-have-quality-scores", + "rowId": "readneed-post-rewrite-benchmark-delta", + "sourcePath": ".bitcode/v41-reading-prompt-benchmark-baselines.json", + "passed": true + }, + { + "id": "readneed-observability-projects-parsed-output", + "rowId": "readneed-post-rewrite-benchmark-delta", + "sourcePath": "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts", + "passed": true + }, + { + "id": "readneed-source-safe-metadata-only", + "rowId": "readneed-post-rewrite-benchmark-delta", + "sourcePath": "sourceSafety", + "passed": true + }, + { + "id": "readfitsfinding-hardening-passing", + "rowId": "readfitsfinding-post-rewrite-benchmark-delta", + "sourcePath": ".bitcode/v41-readfitsfinding-prompt-hardening.json", + "passed": true + }, + { + "id": "readfitsfinding-has-many-candidate-search", + "rowId": "readfitsfinding-post-rewrite-benchmark-delta", + "sourcePath": "packages/pipelines/asset-pack/src/depository-search.ts", + "passed": true + }, + { + "id": "readfitsfinding-embedding-config-present", + "rowId": "readfitsfinding-post-rewrite-benchmark-delta", + "sourcePath": "packages/pipelines/asset-pack/src/embedding-config.ts", + "passed": true + }, + { + "id": "readfitsfinding-search-embeddings-passing", + "rowId": "readfitsfinding-post-rewrite-benchmark-delta", + "sourcePath": "packages/protocol/src/canonical/read-fits-finding-search-embeddings.js", + "passed": true + }, + { + "id": "readfitsfinding-source-safe-preview-only", + "rowId": "readfitsfinding-post-rewrite-benchmark-delta", + "sourcePath": "sourceSafety", + "passed": true + }, + { + "id": "gate7-conversation-artifact-passing", + "rowId": "conversation-interface-post-rewrite-benchmark-delta", + "sourcePath": ".bitcode/v41-conversation-tool-interface-prompt-rewrite.json", + "passed": true + }, + { + "id": "conversation-stream-template-id-only", + "rowId": "conversation-interface-post-rewrite-benchmark-delta", + "sourcePath": "packages/api/src/conversations/stream-events.ts", + "passed": true + }, + { + "id": "conversation-telemetry-redacts-protected-context", + "rowId": "conversation-interface-post-rewrite-benchmark-delta", + "sourcePath": "packages/api/src/conversations/telemetry.ts", + "passed": true + }, + { + "id": "pipeline-log-projects-prompt-result-disclosure", + "rowId": "conversation-interface-post-rewrite-benchmark-delta", + "sourcePath": "uapi/components/base/bitcode/execution/pipeline-execution-log.tsx", + "passed": true + }, + { + "id": "conversation-interface-artifact-source-safe", + "rowId": "conversation-interface-post-rewrite-benchmark-delta", + "sourcePath": "sourceSafety", + "passed": true + }, + { + "id": "gate3-registry-contracts-passing", + "rowId": "registry-lineage-version-telemetry-binding", + "sourcePath": ".bitcode/v41-registry-interpolation-contracts.json", + "passed": true + }, + { + "id": "benchmark-types-have-pbversion-and-quality-gates", + "rowId": "registry-lineage-version-telemetry-binding", + "sourcePath": "packages/prompts/src/benchmarking/types.ts", + "passed": true + }, + { + "id": "benchmark-runner-handles-promptpart-and-prompt", + "rowId": "registry-lineage-version-telemetry-binding", + "sourcePath": "packages/prompts/src/benchmarking/runner.ts", + "passed": true + }, + { + "id": "registry-contracts-bind-hierarchy", + "rowId": "registry-lineage-version-telemetry-binding", + "sourcePath": "packages/protocol/src/canonical/v41-registry-interpolation-contracts.js", + "passed": true + }, + { + "id": "registry-lineage-source-safe", + "rowId": "registry-lineage-version-telemetry-binding", + "sourcePath": "sourceSafety", + "passed": true + }, + { + "id": "v38-ptrr-stack-passing", + "rowId": "failsafe-thricified-inference-receipt-projection", + "sourcePath": "packages/protocol/src/canonical/ptrr-failsafe-thricified-stack.js", + "passed": true + }, + { + "id": "v38-inference-telemetry-passing", + "rowId": "failsafe-thricified-inference-receipt-projection", + "sourcePath": ".bitcode/v38-disclosure-boundary-report.json", + "passed": true + }, + { + "id": "inference-telemetry-has-required-levels", + "rowId": "failsafe-thricified-inference-receipt-projection", + "sourcePath": "packages/protocol/src/canonical/inference-telemetry-disclosure-report.js", + "passed": true + }, + { + "id": "bounded-inference-records-raw-and-parsed", + "rowId": "failsafe-thricified-inference-receipt-projection", + "sourcePath": "packages/pipelines/asset-pack/src/bounded-structured-inference.ts", + "passed": true + }, + { + "id": "observability-projects-generation-receipts", + "rowId": "failsafe-thricified-inference-receipt-projection", + "sourcePath": "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts", + "passed": true + }, + { + "id": "failsafe-thricified-source-safe", + "rowId": "failsafe-thricified-inference-receipt-projection", + "sourcePath": "sourceSafety", + "passed": true + }, + { + "id": "reading-operational-telemetry-passing", + "rowId": "rich-stream-source-safe-telemetry-projection", + "sourcePath": ".bitcode/v39-operational-telemetry-repair-readback.json", + "passed": true + }, + { + "id": "stream-events-sanitize-metadata", + "rowId": "rich-stream-source-safe-telemetry-projection", + "sourcePath": "packages/api/src/conversations/stream-events.ts", + "passed": true + }, + { + "id": "rich-log-header-renders-prompt-result-posture", + "rowId": "rich-stream-source-safe-telemetry-projection", + "sourcePath": "uapi/components/base/bitcode/execution/pipeline-execution-log-header.tsx", + "passed": true + }, + { + "id": "rich-log-tests-cover-telemetry-disclosure", + "rowId": "rich-stream-source-safe-telemetry-projection", + "sourcePath": "uapi/tests/conversationStreamPipelineLog.test.tsx,uapi/tests/pipelineExecutionLogHeader.test.tsx", + "passed": true + }, + { + "id": "rich-stream-source-safe-only", + "rowId": "rich-stream-source-safe-telemetry-projection", + "sourcePath": "sourceSafety", + "passed": true + }, + { + "id": "repair-hooks-present", + "rowId": "repair-hooks-parsed-output-redaction-posture", + "sourcePath": "packages/pipelines/asset-pack/src/reading-operational-telemetry-repair-readback.ts", + "passed": true + }, + { + "id": "repair-readback-redacts-raw-prompts", + "rowId": "repair-hooks-parsed-output-redaction-posture", + "sourcePath": "packages/pipelines/asset-pack/src/reading-operational-telemetry-repair-readback.ts", + "passed": true + }, + { + "id": "conversation-telemetry-redacts-secret-values", + "rowId": "repair-hooks-parsed-output-redaction-posture", + "sourcePath": "packages/api/src/conversations/telemetry.ts", + "passed": true + }, + { + "id": "parsed-output-shape-source-safe", + "rowId": "repair-hooks-parsed-output-redaction-posture", + "sourcePath": "packages/pipelines/asset-pack/src/reading-pipeline-observability.ts", + "passed": true + }, + { + "id": "repair-hook-artifacts-source-safe", + "rowId": "repair-hooks-parsed-output-redaction-posture", + "sourcePath": "sourceSafety", + "passed": true + }, + { + "id": "gate8-source-exists", + "rowId": "gate8-tests-docs-workflows", + "sourcePath": "packages/protocol/src/canonical/v41-prompt-program-benchmark-report.js", + "passed": true + }, + { + "id": "gate8-test-exists", + "rowId": "gate8-tests-docs-workflows", + "sourcePath": "packages/protocol/test/v41-prompt-program-benchmark-report.test.js", + "passed": true + }, + { + "id": "gate8-generator-exists", + "rowId": "gate8-tests-docs-workflows", + "sourcePath": "scripts/generate-v41-prompt-program-benchmark-report.mjs", + "passed": true + }, + { + "id": "gate8-checker-exists", + "rowId": "gate8-tests-docs-workflows", + "sourcePath": "scripts/check-v41-gate8-prompt-program-benchmark-report.mjs", + "passed": true + }, + { + "id": "package-json-exposes-gate8-scripts", + "rowId": "gate8-tests-docs-workflows", + "sourcePath": "package.json", + "passed": true + }, + { + "id": "workflows-run-gate8-checker", + "rowId": "gate8-tests-docs-workflows", + "sourcePath": ".github/workflows/bitcode-gate-quality.yml,.github/workflows/bitcode-canon-quality.yml", + "passed": true + }, + { + "id": "spec-documents-gate8-closure", + "rowId": "gate8-tests-docs-workflows", + "sourcePath": "BITCODE_SPEC_V41.md", + "passed": true + }, + { + "id": "roadmap-documents-gate8-closure", + "rowId": "gate8-tests-docs-workflows", + "sourcePath": "SPECIFICATIONS_ROADMAP.md", + "passed": true + }, + { + "id": "readmes-document-gate8-helpers", + "rowId": "gate8-tests-docs-workflows", + "sourcePath": "README.md,packages/protocol/README.md", + "passed": true + } + ], + "dependencyRoots": { + "v38PromptBenchmarkReportRoot": "v38-prompt-benchmark-report:7c34a5c11c25c490aedf413d", + "v38InferenceTelemetryDisclosureRoot": "v38-inference-telemetry-disclosure-report:78791e4996c6b792217f917c", + "v38PtrrFailsafeThricifiedStackRoot": "v38-ptrr-failsafe-thricified-stack:af3a75b196d4b52e921c0130", + "v38ReadFitsFindingSearchEmbeddingsRoot": "v38-read-fits-finding-search:f3b69abcfaac5d98f30d66d3", + "v39OperationalTelemetryRepairReadbackRoot": "v39-operational-telemetry-repair-readback:bc8e2495bbe81b73a2cdb8dc", + "v40PromptBenchmarkSmokeRoot": "v40-prompt-benchmark-smoke-v41-readiness:3164923746757b9b02fc30ae", + "gate2InventoryRoot": "v41-promptpart-prompt-inventory:296926f1c26d370f7e91a70d", + "gate3RegistryInterpolationRoot": "v41-registry-interpolation-contract:fb4b99ad86f0cf6f4a9c04c4", + "gate4ReadingPromptBenchmarkBaselineRoot": "v41-reading-prompt-benchmark-baselines:82e956682fa1353ed9df6cff", + "gate5ReadNeedPromptHardeningRoot": "v41-readneed-prompt-hardening:e93084e4403cc37d87fca2cf", + "gate6ReadFitsFindingPromptHardeningRoot": "v41-readfitsfinding-prompt-hardening:86e74080708372081079c021", + "gate7ConversationToolInterfacePromptRewriteRoot": "v41-conversation-tool-interface-prompt-rewrite:1c5fab85463bf1064cc5bce5" + }, + "sourceSafety": { + "sourceSafetyClass": "source_safe_prompt_program_benchmark_telemetry_metadata", + "sourceSafeMetadataOnly": true, + "rawPromptTextSerialized": false, + "rawInterpolatedPromptSerialized": false, + "rawProviderResponseSerialized": false, + "protectedPromptSerialized": false, + "protectedSourceVisible": false, + "privateContextSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "walletPrivateMaterialVisible": false, + "settlementPrivatePayloadVisible": false, + "forbiddenPayloadClasses": [ + "secret-values", + "provider-tokens", + "wallet-private-material", + "settlement-private-payloads", + "protected-source-payloads", + "raw-protected-prompts", + "raw-prompt-text", + "raw-interpolated-prompts", + "raw-provider-responses", + "private-context", + "unpaid-assetpack-source" + ] + }, + "coverage": { + "rowCount": 9, + "metricIds": [ + "benchmark_fixture_result_projection", + "depository_search_embedding_query_projection", + "failsafe_thricified_receipt_projection", + "parsed_output_schema_verdict_projection", + "post_rewrite_prompt_delta", + "post_rewrite_promptpart_delta", + "prompt_lineage_registry_versioning", + "repair_hook_redaction_posture", + "rich_stream_prompt_telemetry_projection", + "source_safe_benchmark_report_disclosure" + ], + "metricCount": 10, + "promptProgramArtifactIds": [ + "v38-inference-telemetry-disclosure-report", + "v38-prompt-benchmark-report", + "v38-ptrr-failsafe-thricified-stack", + "v38-read-fits-finding-search-embeddings", + "v39-operational-telemetry-repair-readback", + "v41-conversation-tool-interface-prompt-rewrite", + "v41-prompt-program-benchmark-report", + "v41-promptpart-prompt-inventory", + "v41-readfitsfinding-prompt-hardening", + "v41-reading-prompt-benchmark-baselines", + "v41-readneed-prompt-hardening", + "v41-registry-interpolation-contracts" + ], + "promptProgramArtifactCount": 12, + "telemetryReceiptIds": [ + "AssetPackPreviewBoundary", + "CandidateFits", + "ConversationStreamEvent", + "ConversationTelemetryProofHook", + "PipelineExecutionLog", + "PromptBenchmarkResult", + "PromptPartBenchmarkResult", + "ReadFitsFindingSynthesis", + "ReadNeed", + "ReadNeedComprehensionSynthesisSchema", + "ReadingOperationalTelemetryEvent", + "ReadingPipelineTelemetryProjection", + "RegistryInterpolationReceipt", + "V41PromptProgramBenchmarkReportArtifact", + "agent", + "failsafe", + "interpolated_prompt", + "parsed_output", + "parsed_typed_output_shape_source_safe", + "pipeline_phase", + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "proof_roots_ids_counts_states_and_redacted_error_classes_only", + "ptrr_step", + "raw_response", + "repair", + "schema_verdict", + "thricified_generation" + ], + "telemetryReceiptCount": 28, + "sourceRootCount": 47, + "sourceRootPresentCount": 47, + "missingSourceRoots": [], + "legacySourceRoots": false, + "requiredPredicateCount": 51, + "passedPredicateCount": 51, + "failedPredicateIds": [], + "passingRowCount": 9, + "failingRowIds": [], + "sourceSafeMetadataOnly": true, + "dependenciesPassing": true, + "dependencyRoots": { + "v38PromptBenchmarkReportRoot": "v38-prompt-benchmark-report:7c34a5c11c25c490aedf413d", + "v38InferenceTelemetryDisclosureRoot": "v38-inference-telemetry-disclosure-report:78791e4996c6b792217f917c", + "v38PtrrFailsafeThricifiedStackRoot": "v38-ptrr-failsafe-thricified-stack:af3a75b196d4b52e921c0130", + "v38ReadFitsFindingSearchEmbeddingsRoot": "v38-read-fits-finding-search:f3b69abcfaac5d98f30d66d3", + "v39OperationalTelemetryRepairReadbackRoot": "v39-operational-telemetry-repair-readback:bc8e2495bbe81b73a2cdb8dc", + "v40PromptBenchmarkSmokeRoot": "v40-prompt-benchmark-smoke-v41-readiness:3164923746757b9b02fc30ae", + "gate2InventoryRoot": "v41-promptpart-prompt-inventory:296926f1c26d370f7e91a70d", + "gate3RegistryInterpolationRoot": "v41-registry-interpolation-contract:fb4b99ad86f0cf6f4a9c04c4", + "gate4ReadingPromptBenchmarkBaselineRoot": "v41-reading-prompt-benchmark-baselines:82e956682fa1353ed9df6cff", + "gate5ReadNeedPromptHardeningRoot": "v41-readneed-prompt-hardening:e93084e4403cc37d87fca2cf", + "gate6ReadFitsFindingPromptHardeningRoot": "v41-readfitsfinding-prompt-hardening:86e74080708372081079c021", + "gate7ConversationToolInterfacePromptRewriteRoot": "v41-conversation-tool-interface-prompt-rewrite:1c5fab85463bf1064cc5bce5" + }, + "v38PromptBenchmarkRowCount": 7, + "v38InferenceTelemetryLevelCount": 13, + "v41PromptInventoryRowCount": 0, + "v41ReadingBenchmarkBaselineRowCount": 10, + "v41ReadNeedHardeningRowCount": 7, + "v41ReadFitsFindingHardeningRowCount": 8, + "v41ConversationToolInterfaceRowCount": 9 + }, + "failures": [], + "passed": true +} diff --git a/.bitcode/v41-readfitsfinding-prompt-hardening.json b/.bitcode/v41-readfitsfinding-prompt-hardening.json index 5e46d0f6e..4728caf52 100644 --- a/.bitcode/v41-readfitsfinding-prompt-hardening.json +++ b/.bitcode/v41-readfitsfinding-prompt-hardening.json @@ -6,7 +6,7 @@ "generatedAt": "2026-05-26T00:00:00.000Z", "artifactPath": ".bitcode/v41-readfitsfinding-prompt-hardening.json", "sourceSafetyVerdict": "source-safe-readfitsfinding-prompt-hardening-metadata", - "artifactRoot": "v41-readfitsfinding-prompt-hardening:f5a01c099fd260fca0c081c8", + "artifactRoot": "v41-readfitsfinding-prompt-hardening:86e74080708372081079c021", "passed": true, "failures": [], "metricIds": [ @@ -1301,43 +1301,43 @@ }, { "sourcePath": "BITCODE_SPEC_V41.md", - "sourceHash": "d8c0b4c7f8c268964304aed1" + "sourceHash": "35ed9e60cecdcb8bd820caee" }, { "sourcePath": "BITCODE_SPEC_V41_DELTA.md", - "sourceHash": "1d015ca6f1ee04481adc4d94" + "sourceHash": "68092aae6b6f8916ee5652a0" }, { "sourcePath": "BITCODE_SPEC_V41_NOTES.md", - "sourceHash": "10535375dd3ba572d859cd96" + "sourceHash": "7ed03aa76aa39be60d45ab0c" }, { "sourcePath": "BITCODE_SPEC_V41_PARITY_MATRIX.md", - "sourceHash": "00a808cdce6b067b3b74c284" + "sourceHash": "77e1bfa9c3dd2d549d5123e3" }, { "sourcePath": "SPECIFICATIONS_ROADMAP.md", - "sourceHash": "28927ec4df7539979189a250" + "sourceHash": "ea44c2816a4548954f32a742" }, { "sourcePath": "README.md", - "sourceHash": "ea094a4cf3ceb26af0572f51" + "sourceHash": "64b27506f4fb734632abbb4f" }, { "sourcePath": "packages/protocol/README.md", - "sourceHash": "a350f4f057f5cfd5378e0bde" + "sourceHash": "b755f91f456d78b111c14794" }, { "sourcePath": "package.json", - "sourceHash": "9bd806a6dbff2448f0e5074e" + "sourceHash": "319d175fa7fedbc48b90cb37" }, { "sourcePath": ".github/workflows/bitcode-gate-quality.yml", - "sourceHash": "07c24c49b8ab517a4426ed6b" + "sourceHash": "4498ef474a10506a6cf781ce" }, { "sourcePath": ".github/workflows/bitcode-canon-quality.yml", - "sourceHash": "36d6ce8158e1595685f7856a" + "sourceHash": "75c13714eda5b1bf46ad896e" } ], "sourceRootsPresent": true, diff --git a/.github/workflows/bitcode-canon-quality.yml b/.github/workflows/bitcode-canon-quality.yml index e36fcd2d3..4ba8264f5 100644 --- a/.github/workflows/bitcode-canon-quality.yml +++ b/.github/workflows/bitcode-canon-quality.yml @@ -280,6 +280,9 @@ jobs: if [ -f scripts/check-v41-gate7-conversation-tool-interface-prompt-rewrite.mjs ]; then node scripts/check-v41-gate7-conversation-tool-interface-prompt-rewrite.mjs --skip-branch-check --skip-package-tests fi + if [ -f scripts/check-v41-gate8-prompt-program-benchmark-report.mjs ]; then + node scripts/check-v41-gate8-prompt-program-benchmark-report.mjs --skip-branch-check --skip-package-tests + fi fi else echo "Unexpected BITCODE_SPEC.txt pointer: $POINTER" >&2 diff --git a/.github/workflows/bitcode-gate-quality.yml b/.github/workflows/bitcode-gate-quality.yml index d9cc00ea8..7d518e5d4 100644 --- a/.github/workflows/bitcode-gate-quality.yml +++ b/.github/workflows/bitcode-gate-quality.yml @@ -409,6 +409,9 @@ jobs: if [ -f scripts/check-v41-gate7-conversation-tool-interface-prompt-rewrite.mjs ]; then node scripts/check-v41-gate7-conversation-tool-interface-prompt-rewrite.mjs --skip-branch-check --skip-package-tests fi + if [ -f scripts/check-v41-gate8-prompt-program-benchmark-report.mjs ]; then + node scripts/check-v41-gate8-prompt-program-benchmark-report.mjs --skip-branch-check --skip-package-tests + fi fi else echo "Unexpected BITCODE_SPEC.txt pointer: $POINTER" >&2 diff --git a/BITCODE_SPEC_V41.md b/BITCODE_SPEC_V41.md index 0cf643a03..61af7dba6 100644 --- a/BITCODE_SPEC_V41.md +++ b/BITCODE_SPEC_V41.md @@ -155,6 +155,10 @@ The artifact may include ids, source hashes, counts, prompt surface ids, parser Gate 8 must emit source-safe benchmark and telemetry artifacts after prompt rewrites. It binds benchmark deltas, prompt lineage, prompt registry versions, inference receipts, Failsafe and Thricified receipts, parsed outputs, redaction posture, and repair hooks without serializing raw provider responses or protected prompts. +Gate 8 is closed by the package-backed `buildV41PromptProgramBenchmarkReport` source, deterministic `.bitcode/v41-prompt-program-benchmark-report.json` artifact, `generate:v41-prompt-program-benchmark-report`, `check:v41-prompt-program-benchmark-report`, `check:v41-gate8`, protocol tests, and gate/canon workflow wiring. +The current source-safe benchmark telemetry report covers 9 rows for post-rewrite PromptPart and Prompt deltas, ReadNeedComprehensionSynthesis benchmark deltas, ReadFitsFindingSynthesis benchmark deltas, Conversation/tool/interface prompt deltas, registry lineage and version telemetry, PTRR/Failsafe/Thricified inference receipt projection, rich stream telemetry projection, repair hooks, parsed-output redaction posture, and tests/docs/workflow proof. +The report binds V38 prompt benchmark, V38 inference telemetry disclosure, V38 PTRR/Failsafe/Thricified stack, V38 ReadFitsFinding search embeddings, V39 operational telemetry repair readback, V40 prompt benchmark smoke readiness, and V41 Gate 2 through Gate 7 artifacts. +The artifact may include ids, source hashes, counts, prompt-program artifact ids, telemetry receipt ids, metric ids, predicate verdicts, dependency roots, source-safe benchmark deltas, parser/schema names, and disclosure posture, but no raw prompt text, interpolated prompts, raw provider responses, protected prompts, protected source, private context, credentials, settlement private payloads, wallet private material, or unpaid AssetPack source. ## V41 Gate 9 Promotion Readiness diff --git a/BITCODE_SPEC_V41_DELTA.md b/BITCODE_SPEC_V41_DELTA.md index c48c6138e..27ddfedd9 100644 --- a/BITCODE_SPEC_V41_DELTA.md +++ b/BITCODE_SPEC_V41_DELTA.md @@ -68,6 +68,8 @@ Gate 7 is package-backed by `buildV41ConversationToolInterfacePromptRewrite`, em ### Gate 8: Prompt Benchmark Report And Telemetry Integration Close post-rewrite benchmark deltas, prompt lineage, telemetry integration, and source-safe rich stream projections. +Gate 8 is package-backed by `buildV41PromptProgramBenchmarkReport`, emits `.bitcode/v41-prompt-program-benchmark-report.json`, and covers 9 source-safe rows across post-rewrite PromptPart/Prompt deltas, Reading benchmark deltas, Conversation/tool/interface benchmark deltas, prompt registry lineage, Failsafe and ThricifiedGeneration receipts, parsed-output schema verdicts, rich stream projections, repair hooks, and workflow proof. +It binds V38 prompt benchmark and inference telemetry reports, V38 ReadFitsFinding search embeddings, V39 operational telemetry repair readback, V40 prompt benchmark smoke readiness, and V41 Gate 2 through Gate 7 artifacts without serializing raw prompt text, interpolated prompts, provider responses, protected prompts, protected source, private context, credentials, settlement private payloads, wallet private material, or unpaid AssetPack source. ### Gate 9: Promotion Readiness diff --git a/BITCODE_SPEC_V41_NOTES.md b/BITCODE_SPEC_V41_NOTES.md index a64487e92..8ac2b1b1c 100644 --- a/BITCODE_SPEC_V41_NOTES.md +++ b/BITCODE_SPEC_V41_NOTES.md @@ -8,7 +8,7 @@ - Canonical pointer: `BITCODE_SPEC.txt` -> `V40` - Prior canonical anchor: `BITCODE_SPEC_V40.md` - Prior generated proof appendix: `BITCODE_SPEC_V40_PROVEN.md` -- Generated structured artifact inventory: draft `.bitcode/v41-spec-family-report.json`, draft `.bitcode/v41-canonical-input-report.json`, Gate 2 `.bitcode/v41-promptpart-prompt-inventory.json`, Gate 3 `.bitcode/v41-registry-interpolation-contracts.json`, Gate 4 `.bitcode/v41-reading-prompt-benchmark-baselines.json`, Gate 5 `.bitcode/v41-readneed-prompt-hardening.json`, Gate 6 `.bitcode/v41-readfitsfinding-prompt-hardening.json`, planned conversation rewrite artifact, planned promotion-readiness artifact, and eventual `BITCODE_SPEC_V41_PROVEN.md` after V41 promotion +- Generated structured artifact inventory: draft `.bitcode/v41-spec-family-report.json`, draft `.bitcode/v41-canonical-input-report.json`, Gate 2 `.bitcode/v41-promptpart-prompt-inventory.json`, Gate 3 `.bitcode/v41-registry-interpolation-contracts.json`, Gate 4 `.bitcode/v41-reading-prompt-benchmark-baselines.json`, Gate 5 `.bitcode/v41-readneed-prompt-hardening.json`, Gate 6 `.bitcode/v41-readfitsfinding-prompt-hardening.json`, Gate 7 `.bitcode/v41-conversation-tool-interface-prompt-rewrite.json`, Gate 8 `.bitcode/v41-prompt-program-benchmark-report.json`, planned promotion-readiness artifact, and eventual `BITCODE_SPEC_V41_PROVEN.md` after V41 promotion - Source parity state: V41 notes track prompt-program planning, catalogue, benchmark, rewrite, telemetry, and promotion parity over active V40 - Draft target posture: V41 Gate 1 creates the full SPEC, DELTA, NOTES, and PARITY family. - Scope: Prompt and PromptPart excellence over the promoted V40 exhaustive testing base. @@ -31,6 +31,13 @@ It rewrites ReadFitsFindingSynthesis PromptParts and bounded inference prompts s The package-backed hardening artifact currently emits 7 source-safe rows and 63 passing source predicates across prompt rewrite boundary, PTRR/Failsafe/Thricified composition, strict typed return parsing, source constraints, review/resynthesis, telemetry redaction, and read-comprehension tool prompt alignment. The report is metadata-only: ids, hashes, counts, fixtures, parser targets, predicate verdicts, and dependency roots are allowed; raw prompt payloads, interpolated prompts, provider responses, private context, protected source, credentials, and unpaid AssetPack source remain private. +## Gate 8 implementation notes + +Gate 8 makes the post-rewrite prompt benchmark and telemetry readback source-checkable. +It emits `V41PromptProgramBenchmarkReport` as `.bitcode/v41-prompt-program-benchmark-report.json`, binding V38 prompt benchmark rows, V38 inference telemetry disclosure, V38 PTRR/Failsafe/Thricified stack proof, V38 ReadFitsFinding search embeddings, V39 operational telemetry repair readback, V40 prompt benchmark smoke readiness, and V41 Gate 2 through Gate 7 artifacts. +The report records prompt-program artifact ids, telemetry receipt ids, metric ids, source hashes, source-safe benchmark delta posture, predicate verdicts, and dependency roots only. +It intentionally does not serialize raw prompt text, interpolated prompts, raw provider responses, protected prompts, protected source, private context, credentials, wallet private material, settlement private payloads, or unpaid AssetPack source. + ## Gate 2 prompt inventory note Gate 2 makes the first prompt-program catalogue concrete. diff --git a/BITCODE_SPEC_V41_PARITY_MATRIX.md b/BITCODE_SPEC_V41_PARITY_MATRIX.md index 9188b4977..b351af3f5 100644 --- a/BITCODE_SPEC_V41_PARITY_MATRIX.md +++ b/BITCODE_SPEC_V41_PARITY_MATRIX.md @@ -40,7 +40,7 @@ This matrix records the prompt-program surfaces that must become promotion-grade | ReadNeed rewrite | ReadNeedComprehensionSynthesis prompts rewritten and type-hardened | `.bitcode/v41-readneed-prompt-hardening.json` | implemented | | Finding Fits rewrite | ReadFitsFindingSynthesis prompts rewritten for search and AssetPack context | `.bitcode/v41-readfitsfinding-prompt-hardening.json` | implemented | | Conversation/tool/interface rewrite | Non-Reading prompt surfaces brought to same standard | `.bitcode/v41-conversation-tool-interface-prompt-rewrite.json` | implemented | -| Benchmark and telemetry | Prompt benchmark deltas and telemetry projections source-safe | planned V41 Gate 8 artifact | pending | +| Benchmark and telemetry | Prompt benchmark deltas and telemetry projections source-safe | `.bitcode/v41-prompt-program-benchmark-report.json` | implemented | | Promotion readiness | V41 prompt-program proof and workflow promotion ready | planned V41 Gate 9 artifact | pending | ## V41 implementation checklist @@ -54,7 +54,7 @@ This matrix records the prompt-program surfaces that must become promotion-grade | Gate 5 | ReadNeedComprehensionSynthesis rewrite artifact | implemented | | Gate 6 | ReadFitsFindingSynthesis rewrite artifact | implemented | | Gate 7 | Conversation, tool, and interface prompt rewrite artifact | implemented | -| Gate 8 | Prompt benchmark report and telemetry artifact | pending | +| Gate 8 | Prompt benchmark report and telemetry artifact | implemented | | Gate 9 | Promotion readiness artifact and workflow | pending | ## V41 accepted boundaries diff --git a/README.md b/README.md index 792403c43..421c15bf2 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,15 @@ composition, rich execution-log prompt/result disclosure, DocCodeToolPrompt and ToolPromptRegistry hierarchy, MCP API/public API schemas, ChatGPT App action prompt boundaries, Terminal/public summaries, V38 Conversation/tool parity, and Gate 2 through Gate 6 dependency roots with `check:v41-gate7`. +V41 Gate 8 adds the package-backed `buildV41PromptProgramBenchmarkReport` and +generated source-safe artifact `.bitcode/v41-prompt-program-benchmark-report.json`. +The report binds post-rewrite PromptPart/Prompt deltas, Reading benchmark +baselines, Conversation/tool/interface deltas, prompt registry lineage, +Failsafe and ThricifiedGeneration receipts, parsed-output schema verdicts, rich +stream telemetry, repair hooks, V38/V39/V40 proof roots, and V41 Gate 2 through +Gate 7 dependency roots with `check:v41-gate8`, without serializing raw prompts, +interpolated prompts, provider responses, protected source, or unpaid AssetPack +source. V42 is now roadmapped as the next MVP experience version: shortest-path Depositing with later BTC compensation, shortest-path Reading through Need review/resynthesis, Finding Fits, source-safe AssetPack preview, diff --git a/SPECIFICATIONS_ROADMAP.md b/SPECIFICATIONS_ROADMAP.md index 204a77e29..928a35a5a 100644 --- a/SPECIFICATIONS_ROADMAP.md +++ b/SPECIFICATIONS_ROADMAP.md @@ -5,8 +5,8 @@ - Current active canonical pointer: `BITCODE_SPEC.txt` -> `V40` - Current active canon: `BITCODE_SPEC_V40.md` - Current draft target: `BITCODE_SPEC_V41.md`. -- Current working gate: V41 Gate 7 Conversation Tool And Interface Prompt Rewrite. -- Next queued gate after V41 Gate 7: V41 Prompt Benchmark Report And Telemetry Integration. +- Current working gate: V41 Gate 8 Prompt Benchmark Report And Telemetry Integration. +- Next queued gate after V41 Gate 8: V41 Promotion Readiness. - Latest closed version: V40 Exhaustive Commercial Application Testing, which promoted test inventory, unit coverage, API integration contracts, Reading pipeline integration, Conversation/Terminal integration, browser E2E visual proof, ledger/database/storage/wallet/delivery synchronization, local/staging rehearsal automation, prompt benchmark smoke, and V40 promotion readiness. - Recent V40 closure anchor: V40 canonical promotion updated `BITCODE_SPEC.txt` to `V40`, generated `BITCODE_SPEC_V40_PROVEN.md`, preserved active V40 / draft V41 runtime posture, and closed exhaustive commercial application testing canon. - Recent V39 closure anchor: V39 canonical promotion updated `BITCODE_SPEC.txt` to `V39`, generated `BITCODE_SPEC_V39_PROVEN.md`, preserved active V39 / draft V40 runtime posture, and closed commercial Reading readiness canon. @@ -31,6 +31,7 @@ - V41 Gate 5 closure anchor: prompt-program work now owns package-backed `V41ReadNeedPromptHardening` source, deterministic `.bitcode/v41-readneed-prompt-hardening.json`, 7 source-safe hardening rows, 63 passing source predicates, Gate 2 inventory binding, Gate 3 registry/interpolation binding, Gate 4 benchmark baseline binding, strict ReadNeed return-type checks, review/resynthesis gate proof, source-safe telemetry proof, read-comprehension tool prompt alignment, focused prompt rewrites, protocol tests, workflow wiring, and `check:v41-gate5`. - V41 Gate 6 closure anchor: prompt-program work now owns package-backed `V41ReadFitsFindingPromptHardening` source, deterministic `.bitcode/v41-readfitsfinding-prompt-hardening.json`, 8 source-safe hardening rows, 76 passing source predicates, Gate 2 inventory binding, Gate 3 registry/interpolation binding, Gate 4 Reading benchmark baseline binding, Gate 5 accepted-Need hardening binding, many-candidate Depository search proof, embeddings and provider-ranking policy proof, selected-fit provenance roots, AssetPack context synthesis boundaries, source-safe preview and quote disclosure, post-settlement BTD rights and delivery posture, focused prompt rewrites, protocol tests, workflow wiring, and `check:v41-gate6`. - V41 Gate 7 closure anchor: prompt-program work now owns package-backed `V41ConversationToolInterfacePromptRewrite` source, deterministic `.bitcode/v41-conversation-tool-interface-prompt-rewrite.json`, 9 source-safe rewrite rows, 60 passing source predicates, Conversation PTRR PromptPart rewrites, Terminal conversation system prompt boundaries, rich execution-log prompt/result disclosure, DocCodeToolPrompt and ToolPromptRegistry hierarchy, MCP API/public API contract prompt posture, ChatGPT App action/tool prompt posture, Terminal/public summary source-safety, Gate 2 through Gate 6 dependency roots, V38 Conversation/tool parity binding, protocol tests, workflow wiring, and `check:v41-gate7`. +- V41 Gate 8 closure anchor: prompt-program work now owns package-backed `V41PromptProgramBenchmarkReport` source, deterministic `.bitcode/v41-prompt-program-benchmark-report.json`, 9 source-safe benchmark telemetry rows, post-rewrite PromptPart and Prompt deltas, benchmark fixture projections, prompt registry lineage, V38 PromptBenchmark/Failsafe/Thricified/inference telemetry roots, V39 operational repair readback roots, V40 prompt benchmark smoke roots, V41 Gate 2 through Gate 7 dependency roots, rich execution-log telemetry projections, repair hook and parsed-output redaction posture, protocol tests, workflow wiring, and `check:v41-gate8`. - Forward planning note: V41 should focus singularly on Prompt and PromptPart implementation quality: prompts as programs. V38's inference correctness hardening and V40's testing/benchmark depth should empower V41 to examine every raw PromptPart, every composed Prompt, every benchmark, every meaningfully benchmarkable semantic part, title, template, interpolation contract, registry binding, inference callsite, and downstream parsed return type; then repartition, retitle, rewrite, catalogue, and validate them across Reading pipelines and conversational interactions. V41 should primarily harden `ReadNeedComprehensionSynthesis` and `ReadFitsFindingSynthesis` prompt surfaces while also covering conversational and other inference prompts with the same catalogue-and-benchmark discipline. - Forward planning note: V42 should focus on the reliable MVP product experience after V41's prompt-program hardening. Depositing should become the shortest path to adding any source material, proving its Depository admission, and later receiving BTC compensation when that deposit contributes to a synthesized AssetPack. Reading should become the shortest path to submitting a Read Request, reviewing or resynthesizing Bitcode's synthesized Need, requesting Finding Fits, reviewing source-safe AssetPack measurements and preview metadata, buying the AssetPack through BTD/BTC settlement semantics, and receiving post-settlement repository delivery. The demonstration should be AI-reading dominant: any deposit source can contribute proprietary or otherwise non-public training, prompt, context, or evaluation material to an AssetPack that measurably improves an AI system beyond a public-data-only baseline. - Forward planning note: V43+ should evolve the deposit side into an agentic AssetPack option experience for enterprises that own connected codebases. Bitcode Agents installed on an enterprise repository should compare the repository, the current Bitcode Depository, and current Reading activity to propose deposit AssetPack options: unminted AssetPacks in all but BTD that can later become BTD only when a Reader's industrial Need-Fit mints one. The deposit pipeline should optimize for enterprise IP governance by filtering out critical IP, estimating whether selling would be positive ROI against development cost and expected Read demand, and presenting everything else as approve/reject deposit options. The product route plan for that version is to split `/terminal` into `/read` for Reading and `/deposit` for agentic deposit AssetPack option review, and to rename `/exchange` to `/packs` across routes, code names, and docs. diff --git a/package.json b/package.json index a59c5ca69..eab3a797c 100644 --- a/package.json +++ b/package.json @@ -297,6 +297,9 @@ "generate:v41-conversation-tool-interface-prompt-rewrite": "node scripts/generate-v41-conversation-tool-interface-prompt-rewrite.mjs", "check:v41-conversation-tool-interface-prompt-rewrite": "node scripts/generate-v41-conversation-tool-interface-prompt-rewrite.mjs --check", "check:v41-gate7": "node scripts/check-v41-gate7-conversation-tool-interface-prompt-rewrite.mjs", + "generate:v41-prompt-program-benchmark-report": "node scripts/generate-v41-prompt-program-benchmark-report.mjs", + "check:v41-prompt-program-benchmark-report": "node scripts/generate-v41-prompt-program-benchmark-report.mjs --check", + "check:v41-gate8": "node scripts/check-v41-gate8-prompt-program-benchmark-report.mjs", "generate:v38-inference-surface-inventory": "node scripts/generate-v38-inference-surface-inventory.mjs", "check:v38-inference-surface-inventory": "node scripts/generate-v38-inference-surface-inventory.mjs --check", "check:v38-gate2": "node scripts/check-v38-gate2-inference-surface-inventory.mjs", diff --git a/packages/protocol/README.md b/packages/protocol/README.md index 98065004d..f1d41bc7e 100644 --- a/packages/protocol/README.md +++ b/packages/protocol/README.md @@ -150,6 +150,18 @@ DocCodeToolPrompt and ToolPromptRegistry hierarchy, MCP API/public API tool schema posture, ChatGPT App action/tool prompt posture, Terminal/public summary source-safety, V38 Conversation/tool parity, and Gate 2 through Gate 6 dependency roots. +V41 Gate 8 adds `V41PromptProgramBenchmarkReport` through +`packages/protocol/src/canonical/v41-prompt-program-benchmark-report.js`, +`packages/protocol/test/v41-prompt-program-benchmark-report.test.js`, +`.bitcode/v41-prompt-program-benchmark-report.json`, +`generate:v41-prompt-program-benchmark-report`, +`check:v41-prompt-program-benchmark-report`, and `check:v41-gate8`. +The artifact is source-safe metadata only and currently binds 9 benchmark +telemetry rows across post-rewrite prompt deltas, Reading benchmark deltas, +Conversation/tool/interface deltas, prompt registry lineage, Failsafe and +ThricifiedGeneration receipts, parsed-output schema verdicts, rich stream +telemetry, repair hooks, and V38/V39/V40 plus V41 Gate 2 through Gate 7 +dependency roots. V40 Gate 2 adds `V40TestInventoryCoverageMatrix` through `packages/protocol/src/canonical/v40-test-inventory-coverage-matrix.js`, `packages/protocol/test/v40-test-inventory-coverage-matrix.test.js`, diff --git a/packages/protocol/src/canonical/v41-conversation-tool-interface-prompt-rewrite.js b/packages/protocol/src/canonical/v41-conversation-tool-interface-prompt-rewrite.js index 810177c8a..cf3f5dee7 100644 --- a/packages/protocol/src/canonical/v41-conversation-tool-interface-prompt-rewrite.js +++ b/packages/protocol/src/canonical/v41-conversation-tool-interface-prompt-rewrite.js @@ -866,8 +866,8 @@ function buildPredicateResults(repoRoot) { 'conversation-tool-interface-tests-docs-workflows', SOURCE_ROOTS.roadmap, roadmap.includes('V41 Gate 7 closure anchor') && - roadmap.includes('Current working gate: V41 Gate 7') && - roadmap.includes('Next queued gate after V41 Gate 7'), + /Current working gate: V41 Gate (?:7|8|9)/u.test(roadmap) && + /Next queued gate after V41 Gate (?:7|8): V41 (?:Prompt Benchmark Report And Telemetry Integration|Promotion Readiness)/u.test(roadmap), ), predicate( 'readmes-document-gate7-helpers', diff --git a/packages/protocol/src/canonical/v41-prompt-program-benchmark-report.js b/packages/protocol/src/canonical/v41-prompt-program-benchmark-report.js new file mode 100644 index 000000000..29204d060 --- /dev/null +++ b/packages/protocol/src/canonical/v41-prompt-program-benchmark-report.js @@ -0,0 +1,1010 @@ +// @ts-check + +import crypto from 'node:crypto'; +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { buildV38InferenceTelemetryDisclosureReport } from './inference-telemetry-disclosure-report.js'; +import { buildV38PromptBenchmarkReport } from './prompt-benchmark-report.js'; +import { buildV38PtrrFailsafeThricifiedStack } from './ptrr-failsafe-thricified-stack.js'; +import { buildV38ReadFitsFindingSearchEmbeddings } from './read-fits-finding-search-embeddings.js'; +import { buildV39OperationalTelemetryRepairReadback } from './v39-operational-telemetry-repair-readback.js'; +import { buildV40PromptBenchmarkSmokeV41Readiness } from './v40-prompt-benchmark-smoke-v41-readiness.js'; +import { buildV41ConversationToolInterfacePromptRewrite } from './v41-conversation-tool-interface-prompt-rewrite.js'; +import { buildV41PromptPartPromptInventory } from './v41-promptpart-prompt-inventory.js'; +import { buildV41ReadFitsFindingPromptHardening } from './v41-readfitsfinding-prompt-hardening.js'; +import { buildV41ReadNeedPromptHardening } from './v41-readneed-prompt-hardening.js'; +import { buildV41ReadingPromptBenchmarkBaselines } from './v41-reading-prompt-benchmark-baselines.js'; +import { buildV41RegistryInterpolationContracts } from './v41-registry-interpolation-contracts.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const DEFAULT_REPO_ROOT = path.resolve(__dirname, '..', '..', '..', '..'); + +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ARTIFACT_PATH = + '.bitcode/v41-prompt-program-benchmark-report.json'; +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_SCHEMA_ID = + 'bitcode.v41.promptProgramBenchmarkReport.v1'; +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_VERSION = 'V41'; +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_CURRENT_TARGET = 'V40'; +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_SOURCE_SAFETY_VERDICT = + 'source-safe-prompt-program-benchmark-telemetry-metadata'; + +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_METRIC_IDS = Object.freeze([ + 'post_rewrite_promptpart_delta', + 'post_rewrite_prompt_delta', + 'benchmark_fixture_result_projection', + 'prompt_lineage_registry_versioning', + 'failsafe_thricified_receipt_projection', + 'parsed_output_schema_verdict_projection', + 'rich_stream_prompt_telemetry_projection', + 'repair_hook_redaction_posture', + 'source_safe_benchmark_report_disclosure', + 'depository_search_embedding_query_projection', +]); + +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_DISCLOSURE_TIERS = Object.freeze([ + 'prompt_identity_source_safe', + 'promptpart_identity_source_safe', + 'prompt_registry_lineage_source_safe', + 'benchmark_fixture_id_source_safe', + 'benchmark_delta_score_source_safe', + 'telemetry_event_identity_source_safe', + 'failsafe_receipt_identity_source_safe', + 'thricified_generation_identity_source_safe', + 'parsed_output_shape_source_safe', + 'schema_verdict_source_safe', + 'repair_hook_identity_source_safe', + 'source_hash_source_safe', + 'raw_prompt_text_private', + 'raw_interpolated_prompt_private', + 'raw_provider_response_private', + 'protected_prompt_private', + 'protected_source_private', + 'private_context_private', + 'unpaid_assetpack_source_private', + 'wallet_private_material_private', + 'settlement_private_payload_private', +]); + +const FORBIDDEN_PAYLOAD_CLASSES = Object.freeze([ + 'secret-values', + 'provider-tokens', + 'wallet-private-material', + 'settlement-private-payloads', + 'protected-source-payloads', + 'raw-protected-prompts', + 'raw-prompt-text', + 'raw-interpolated-prompts', + 'raw-provider-responses', + 'private-context', + 'unpaid-assetpack-source', +]); + +const SOURCE_ROOTS = Object.freeze({ + promptBenchmarkRunner: 'packages/prompts/src/benchmarking/runner.ts', + promptBenchmarkTypes: 'packages/prompts/src/benchmarking/types.ts', + promptBenchmarkReadme: 'packages/prompts/src/benchmarking/README.md', + readingPipelineContract: 'packages/pipelines/asset-pack/src/reading-pipeline-contract.ts', + readingObservability: 'packages/pipelines/asset-pack/src/reading-pipeline-observability.ts', + readingOperationalTelemetry: 'packages/pipelines/asset-pack/src/reading-operational-telemetry-repair-readback.ts', + boundedStructuredInference: 'packages/pipelines/asset-pack/src/bounded-structured-inference.ts', + depositorySearch: 'packages/pipelines/asset-pack/src/depository-search.ts', + depositorySupplyIndex: 'packages/pipelines/asset-pack/src/depository-supply-index.ts', + embeddingConfig: 'packages/pipelines/asset-pack/src/embedding-config.ts', + readFitsRuntime: 'packages/pipelines/asset-pack/src/read-fits-finding-runtime.ts', + conversationStreamEvents: 'packages/api/src/conversations/stream-events.ts', + conversationTelemetry: 'packages/api/src/conversations/telemetry.ts', + pipelineLog: 'uapi/components/base/bitcode/execution/pipeline-execution-log.tsx', + pipelineLogHeader: 'uapi/components/base/bitcode/execution/pipeline-execution-log-header.tsx', + pipelineLogTest: 'uapi/tests/conversationStreamPipelineLog.test.tsx', + pipelineLogHeaderTest: 'uapi/tests/pipelineExecutionLogHeader.test.tsx', + v38PromptBenchmarkSource: 'packages/protocol/src/canonical/prompt-benchmark-report.js', + v38PromptBenchmarkArtifact: '.bitcode/v38-prompt-benchmark-report.json', + v38InferenceTelemetrySource: 'packages/protocol/src/canonical/inference-telemetry-disclosure-report.js', + v38InferenceTelemetryArtifact: '.bitcode/v38-disclosure-boundary-report.json', + v38PtrrStackSource: 'packages/protocol/src/canonical/ptrr-failsafe-thricified-stack.js', + v38ReadFitsSearchEmbeddingsSource: 'packages/protocol/src/canonical/read-fits-finding-search-embeddings.js', + v39OperationalTelemetrySource: 'packages/protocol/src/canonical/v39-operational-telemetry-repair-readback.js', + v39OperationalTelemetryArtifact: '.bitcode/v39-operational-telemetry-repair-readback.json', + v40PromptBenchmarkSmokeSource: 'packages/protocol/src/canonical/v40-prompt-benchmark-smoke-v41-readiness.js', + v40PromptBenchmarkSmokeArtifact: '.bitcode/v40-prompt-benchmark-smoke-v41-readiness.json', + gate2InventorySource: 'packages/protocol/src/canonical/v41-promptpart-prompt-inventory.js', + gate2InventoryArtifact: '.bitcode/v41-promptpart-prompt-inventory.json', + gate3ContractsSource: 'packages/protocol/src/canonical/v41-registry-interpolation-contracts.js', + gate3ContractsArtifact: '.bitcode/v41-registry-interpolation-contracts.json', + gate4BaselinesSource: 'packages/protocol/src/canonical/v41-reading-prompt-benchmark-baselines.js', + gate4BaselinesArtifact: '.bitcode/v41-reading-prompt-benchmark-baselines.json', + gate5ReadNeedSource: 'packages/protocol/src/canonical/v41-readneed-prompt-hardening.js', + gate5ReadNeedArtifact: '.bitcode/v41-readneed-prompt-hardening.json', + gate6ReadFitsSource: 'packages/protocol/src/canonical/v41-readfitsfinding-prompt-hardening.js', + gate6ReadFitsArtifact: '.bitcode/v41-readfitsfinding-prompt-hardening.json', + gate7ConversationSource: 'packages/protocol/src/canonical/v41-conversation-tool-interface-prompt-rewrite.js', + gate7ConversationArtifact: '.bitcode/v41-conversation-tool-interface-prompt-rewrite.json', + packageSource: 'packages/protocol/src/canonical/v41-prompt-program-benchmark-report.js', + packageTest: 'packages/protocol/test/v41-prompt-program-benchmark-report.test.js', + generator: 'scripts/generate-v41-prompt-program-benchmark-report.mjs', + checker: 'scripts/check-v41-gate8-prompt-program-benchmark-report.mjs', + spec: 'BITCODE_SPEC_V41.md', + delta: 'BITCODE_SPEC_V41_DELTA.md', + notes: 'BITCODE_SPEC_V41_NOTES.md', + parity: 'BITCODE_SPEC_V41_PARITY_MATRIX.md', + roadmap: 'SPECIFICATIONS_ROADMAP.md', + readme: 'README.md', + protocolReadme: 'packages/protocol/README.md', + packageJson: 'package.json', + gateWorkflow: '.github/workflows/bitcode-gate-quality.yml', + canonWorkflow: '.github/workflows/bitcode-canon-quality.yml', +}); + +function digest(value) { + return crypto.createHash('sha256').update(String(value)).digest('hex').slice(0, 24); +} + +function artifactRoot(input) { + return `v41-prompt-program-benchmark-report:${digest(input)}`; +} + +function rowRoot(input) { + return `v41-prompt-program-benchmark-report-row:${digest(input)}`; +} + +function sourceExists(repoRoot, sourcePath) { + return existsSync(path.join(repoRoot, sourcePath)); +} + +function readSource(repoRoot, sourcePath) { + const absolutePath = path.join(repoRoot, sourcePath); + return existsSync(absolutePath) ? readFileSync(absolutePath, 'utf8') : ''; +} + +function listFiles(repoRoot, sourcePath) { + const absolutePath = path.join(repoRoot, sourcePath); + if (!existsSync(absolutePath)) return []; + const stat = statSync(absolutePath); + if (stat.isFile()) return [sourcePath]; + if (!stat.isDirectory()) return []; + + const files = []; + const walk = (currentAbsolute, currentRelative) => { + const entries = readdirSync(currentAbsolute, { withFileTypes: true }); + const entryNames = new Set(entries.map((entry) => entry.name)); + for (const entry of entries) { + if (entry.name === 'dist' || entry.name === 'node_modules' || entry.name === '_legacy') continue; + const nextAbsolute = path.join(currentAbsolute, entry.name); + const nextRelative = path.join(currentRelative, entry.name); + if (entry.isDirectory()) { + walk(nextAbsolute, nextRelative); + continue; + } + if (entry.name.endsWith('.js') && entryNames.has(`${entry.name.slice(0, -3)}.ts`)) continue; + if (/\.(?:ts|tsx|js|mjs|md|json|yml)$/u.test(entry.name) && !/\.d\.ts$/u.test(entry.name)) { + files.push(nextRelative); + } + } + }; + walk(absolutePath, sourcePath); + return files.sort(); +} + +function sourceHash(repoRoot, sourcePath) { + const files = listFiles(repoRoot, sourcePath); + const input = files.map((file) => `${file}:${digest(readSource(repoRoot, file))}`).join('|'); + return digest(input); +} + +function unique(values) { + return [...new Set(values.filter(Boolean))].sort(); +} + +function predicate(id, rowId, sourcePath, passed) { + return { id, rowId, sourcePath, passed: Boolean(passed) }; +} + +function row(input) { + return { + ...input, + rowRoot: rowRoot(input.reportRowId), + benchmarkReportMode: 'post-rewrite-source-safe-metadata', + sourceSafetyClass: 'source_safe_prompt_program_benchmark_telemetry_metadata', + sourceSafeMetadataOnly: true, + disclosureTiers: [...V41_PROMPT_PROGRAM_BENCHMARK_REPORT_DISCLOSURE_TIERS], + rawPromptTextSerialized: false, + rawInterpolatedPromptSerialized: false, + rawProviderResponseSerialized: false, + protectedPromptSerialized: false, + protectedSourceVisible: false, + privateContextSerialized: false, + credentialsSerialized: false, + unpaidAssetPackSourceVisible: false, + walletPrivateMaterialVisible: false, + settlementPrivatePayloadVisible: false, + forbiddenPayloadClasses: [...FORBIDDEN_PAYLOAD_CLASSES], + }; +} + +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ROWS = Object.freeze([ + row({ + reportRowId: 'post-rewrite-prompt-inventory-delta', + label: 'Gate 2 prompt inventory deltas are re-bound after Reading and Conversation prompt rewrites', + sourceRoots: [ + SOURCE_ROOTS.gate2InventorySource, + SOURCE_ROOTS.gate2InventoryArtifact, + SOURCE_ROOTS.gate5ReadNeedArtifact, + SOURCE_ROOTS.gate6ReadFitsArtifact, + SOURCE_ROOTS.gate7ConversationArtifact, + ], + promptProgramArtifactIds: [ + 'v41-promptpart-prompt-inventory', + 'v41-readneed-prompt-hardening', + 'v41-readfitsfinding-prompt-hardening', + 'v41-conversation-tool-interface-prompt-rewrite', + ], + metricIds: ['post_rewrite_promptpart_delta', 'post_rewrite_prompt_delta', 'source_safe_benchmark_report_disclosure'], + telemetryReceiptIds: ['promptpart_identity_source_safe', 'prompt_identity_source_safe'], + requiredPredicateIds: [ + 'gate2-inventory-passing', + 'gate5-readneed-passing', + 'gate6-readfitsfinding-passing', + 'gate7-conversation-interface-passing', + 'post-rewrite-artifacts-present', + 'post-rewrite-report-does-not-serialize-raw-prompt-text', + ], + }), + row({ + reportRowId: 'readneed-post-rewrite-benchmark-delta', + label: 'ReadNeedComprehensionSynthesis benchmark deltas bind prompt revisions to typed Need outputs', + sourceRoots: [ + SOURCE_ROOTS.gate4BaselinesSource, + SOURCE_ROOTS.gate4BaselinesArtifact, + SOURCE_ROOTS.gate5ReadNeedSource, + SOURCE_ROOTS.gate5ReadNeedArtifact, + SOURCE_ROOTS.readingPipelineContract, + SOURCE_ROOTS.readingObservability, + ], + promptProgramArtifactIds: [ + 'v41-reading-prompt-benchmark-baselines', + 'v41-readneed-prompt-hardening', + ], + metricIds: [ + 'benchmark_fixture_result_projection', + 'parsed_output_schema_verdict_projection', + 'prompt_lineage_registry_versioning', + ], + telemetryReceiptIds: ['ReadNeed', 'ReadNeedComprehensionSynthesisSchema', 'parsed_typed_output_shape_source_safe'], + requiredPredicateIds: [ + 'gate4-reading-benchmark-passing', + 'readneed-hardening-row-count', + 'readneed-benchmark-baselines-have-quality-scores', + 'readneed-observability-projects-parsed-output', + 'readneed-source-safe-metadata-only', + ], + }), + row({ + reportRowId: 'readfitsfinding-post-rewrite-benchmark-delta', + label: 'ReadFitsFindingSynthesis benchmark deltas bind many-candidate search, embeddings, and AssetPack preview metadata', + sourceRoots: [ + SOURCE_ROOTS.gate4BaselinesSource, + SOURCE_ROOTS.gate4BaselinesArtifact, + SOURCE_ROOTS.gate6ReadFitsSource, + SOURCE_ROOTS.gate6ReadFitsArtifact, + SOURCE_ROOTS.v38ReadFitsSearchEmbeddingsSource, + SOURCE_ROOTS.depositorySearch, + SOURCE_ROOTS.embeddingConfig, + SOURCE_ROOTS.readFitsRuntime, + ], + promptProgramArtifactIds: [ + 'v41-reading-prompt-benchmark-baselines', + 'v41-readfitsfinding-prompt-hardening', + 'v38-read-fits-finding-search-embeddings', + ], + metricIds: [ + 'benchmark_fixture_result_projection', + 'depository_search_embedding_query_projection', + 'parsed_output_schema_verdict_projection', + ], + telemetryReceiptIds: ['ReadFitsFindingSynthesis', 'CandidateFits', 'AssetPackPreviewBoundary'], + requiredPredicateIds: [ + 'readfitsfinding-hardening-passing', + 'readfitsfinding-has-many-candidate-search', + 'readfitsfinding-embedding-config-present', + 'readfitsfinding-search-embeddings-passing', + 'readfitsfinding-source-safe-preview-only', + ], + }), + row({ + reportRowId: 'conversation-interface-post-rewrite-benchmark-delta', + label: 'Conversation, tool-definition, MCP API, ChatGPT App, and Terminal prompt rewrites bind to benchmark metadata', + sourceRoots: [ + SOURCE_ROOTS.gate7ConversationSource, + SOURCE_ROOTS.gate7ConversationArtifact, + SOURCE_ROOTS.conversationStreamEvents, + SOURCE_ROOTS.conversationTelemetry, + SOURCE_ROOTS.pipelineLog, + SOURCE_ROOTS.pipelineLogHeader, + ], + promptProgramArtifactIds: ['v41-conversation-tool-interface-prompt-rewrite'], + metricIds: [ + 'benchmark_fixture_result_projection', + 'rich_stream_prompt_telemetry_projection', + 'source_safe_benchmark_report_disclosure', + ], + telemetryReceiptIds: ['ConversationStreamEvent', 'ConversationTelemetryProofHook', 'PipelineExecutionLog'], + requiredPredicateIds: [ + 'gate7-conversation-artifact-passing', + 'conversation-stream-template-id-only', + 'conversation-telemetry-redacts-protected-context', + 'pipeline-log-projects-prompt-result-disclosure', + 'conversation-interface-artifact-source-safe', + ], + }), + row({ + reportRowId: 'registry-lineage-version-telemetry-binding', + label: 'Prompt registry lineage and version metadata are projected as ids, roots, counts, and schema names', + sourceRoots: [ + SOURCE_ROOTS.gate3ContractsSource, + SOURCE_ROOTS.gate3ContractsArtifact, + SOURCE_ROOTS.promptBenchmarkTypes, + SOURCE_ROOTS.promptBenchmarkRunner, + SOURCE_ROOTS.readingPipelineContract, + ], + promptProgramArtifactIds: ['v41-registry-interpolation-contracts', 'v38-prompt-benchmark-report'], + metricIds: [ + 'prompt_lineage_registry_versioning', + 'post_rewrite_promptpart_delta', + 'post_rewrite_prompt_delta', + ], + telemetryReceiptIds: ['PromptPartBenchmarkResult', 'PromptBenchmarkResult', 'RegistryInterpolationReceipt'], + requiredPredicateIds: [ + 'gate3-registry-contracts-passing', + 'benchmark-types-have-pbversion-and-quality-gates', + 'benchmark-runner-handles-promptpart-and-prompt', + 'registry-contracts-bind-hierarchy', + 'registry-lineage-source-safe', + ], + }), + row({ + reportRowId: 'failsafe-thricified-inference-receipt-projection', + label: 'PTRR steps, Failsafe sequences, ThricifiedGeneration calls, raw-output roots, and parsed outputs remain traceable', + sourceRoots: [ + SOURCE_ROOTS.v38PtrrStackSource, + SOURCE_ROOTS.v38InferenceTelemetrySource, + SOURCE_ROOTS.v38InferenceTelemetryArtifact, + SOURCE_ROOTS.boundedStructuredInference, + SOURCE_ROOTS.readingObservability, + ], + promptProgramArtifactIds: [ + 'v38-ptrr-failsafe-thricified-stack', + 'v38-inference-telemetry-disclosure-report', + ], + metricIds: [ + 'failsafe_thricified_receipt_projection', + 'parsed_output_schema_verdict_projection', + 'rich_stream_prompt_telemetry_projection', + ], + telemetryReceiptIds: [ + 'pipeline_phase', + 'agent', + 'ptrr_step', + 'failsafe', + 'thricified_generation', + 'interpolated_prompt', + 'raw_response', + 'parsed_output', + 'schema_verdict', + ], + requiredPredicateIds: [ + 'v38-ptrr-stack-passing', + 'v38-inference-telemetry-passing', + 'inference-telemetry-has-required-levels', + 'bounded-inference-records-raw-and-parsed', + 'observability-projects-generation-receipts', + 'failsafe-thricified-source-safe', + ], + }), + row({ + reportRowId: 'rich-stream-source-safe-telemetry-projection', + label: 'Rich execution-log streams expose prompt and result posture as source-safe metadata for live operator inspection', + sourceRoots: [ + SOURCE_ROOTS.conversationStreamEvents, + SOURCE_ROOTS.conversationTelemetry, + SOURCE_ROOTS.readingObservability, + SOURCE_ROOTS.readingOperationalTelemetry, + SOURCE_ROOTS.pipelineLog, + SOURCE_ROOTS.pipelineLogHeader, + SOURCE_ROOTS.pipelineLogTest, + SOURCE_ROOTS.pipelineLogHeaderTest, + ], + promptProgramArtifactIds: ['v38-inference-telemetry-disclosure-report', 'v39-operational-telemetry-repair-readback'], + metricIds: [ + 'rich_stream_prompt_telemetry_projection', + 'repair_hook_redaction_posture', + 'source_safe_benchmark_report_disclosure', + ], + telemetryReceiptIds: ['ReadingPipelineTelemetryProjection', 'ReadingOperationalTelemetryEvent', 'ConversationStreamEvent'], + requiredPredicateIds: [ + 'reading-operational-telemetry-passing', + 'stream-events-sanitize-metadata', + 'rich-log-header-renders-prompt-result-posture', + 'rich-log-tests-cover-telemetry-disclosure', + 'rich-stream-source-safe-only', + ], + }), + row({ + reportRowId: 'repair-hooks-parsed-output-redaction-posture', + label: 'Repair hooks and parsed output projections remain debuggable without leaking source, prompts, responses, or settlement payloads', + sourceRoots: [ + SOURCE_ROOTS.readingOperationalTelemetry, + SOURCE_ROOTS.v39OperationalTelemetrySource, + SOURCE_ROOTS.v39OperationalTelemetryArtifact, + SOURCE_ROOTS.readingObservability, + SOURCE_ROOTS.conversationTelemetry, + ], + promptProgramArtifactIds: ['v39-operational-telemetry-repair-readback', 'v38-inference-telemetry-disclosure-report'], + metricIds: [ + 'repair_hook_redaction_posture', + 'parsed_output_schema_verdict_projection', + 'source_safe_benchmark_report_disclosure', + ], + telemetryReceiptIds: ['repair', 'schema_verdict', 'proof_roots_ids_counts_states_and_redacted_error_classes_only'], + requiredPredicateIds: [ + 'repair-hooks-present', + 'repair-readback-redacts-raw-prompts', + 'conversation-telemetry-redacts-secret-values', + 'parsed-output-shape-source-safe', + 'repair-hook-artifacts-source-safe', + ], + }), + row({ + reportRowId: 'gate8-tests-docs-workflows', + label: 'Gate 8 tests, scripts, docs, generated artifact, package exports, and workflows close benchmark telemetry integration', + sourceRoots: [ + SOURCE_ROOTS.packageSource, + SOURCE_ROOTS.packageTest, + SOURCE_ROOTS.generator, + SOURCE_ROOTS.checker, + SOURCE_ROOTS.spec, + SOURCE_ROOTS.delta, + SOURCE_ROOTS.notes, + SOURCE_ROOTS.parity, + SOURCE_ROOTS.roadmap, + SOURCE_ROOTS.readme, + SOURCE_ROOTS.protocolReadme, + SOURCE_ROOTS.packageJson, + SOURCE_ROOTS.gateWorkflow, + SOURCE_ROOTS.canonWorkflow, + ], + promptProgramArtifactIds: ['v41-prompt-program-benchmark-report'], + metricIds: ['source_safe_benchmark_report_disclosure', 'repair_hook_redaction_posture'], + telemetryReceiptIds: ['V41PromptProgramBenchmarkReportArtifact'], + requiredPredicateIds: [ + 'gate8-source-exists', + 'gate8-test-exists', + 'gate8-generator-exists', + 'gate8-checker-exists', + 'package-json-exposes-gate8-scripts', + 'workflows-run-gate8-checker', + 'spec-documents-gate8-closure', + 'roadmap-documents-gate8-closure', + 'readmes-document-gate8-helpers', + ], + }), +]); + +function dependencyContext(generatedAt, repoRoot) { + const v38PromptBenchmark = buildV38PromptBenchmarkReport({ generatedAt, repoRoot }); + const v38InferenceTelemetry = buildV38InferenceTelemetryDisclosureReport({ generatedAt, repoRoot }); + const v38PtrrStack = buildV38PtrrFailsafeThricifiedStack({ generatedAt, repoRoot }); + const v38ReadFitsSearchEmbeddings = buildV38ReadFitsFindingSearchEmbeddings({ generatedAt, repoRoot }); + const v39OperationalTelemetry = buildV39OperationalTelemetryRepairReadback({ generatedAt, repoRoot }); + const v40PromptBenchmarkSmoke = buildV40PromptBenchmarkSmokeV41Readiness({ generatedAt, repoRoot }); + const gate2Inventory = buildV41PromptPartPromptInventory({ generatedAt, repoRoot }); + const gate3Contracts = buildV41RegistryInterpolationContracts({ generatedAt, repoRoot }); + const gate4Baselines = buildV41ReadingPromptBenchmarkBaselines({ generatedAt, repoRoot }); + const gate5ReadNeed = buildV41ReadNeedPromptHardening({ generatedAt, repoRoot }); + const gate6ReadFits = buildV41ReadFitsFindingPromptHardening({ generatedAt, repoRoot }); + const gate7Conversation = buildV41ConversationToolInterfacePromptRewrite({ generatedAt, repoRoot }); + + return { + v38PromptBenchmark, + v38InferenceTelemetry, + v38PtrrStack, + v38ReadFitsSearchEmbeddings, + v39OperationalTelemetry, + v40PromptBenchmarkSmoke, + gate2Inventory, + gate3Contracts, + gate4Baselines, + gate5ReadNeed, + gate6ReadFits, + gate7Conversation, + }; +} + +function buildSourceEvidence(repoRoot, rows) { + return unique(rows.flatMap((item) => item.sourceRoots)).map((sourcePath) => { + const files = listFiles(repoRoot, sourcePath); + return { + sourcePath, + exists: sourceExists(repoRoot, sourcePath), + legacy: sourcePath.startsWith('_legacy/'), + fileCount: files.length, + sourceRootHash: files.length ? sourceHash(repoRoot, sourcePath) : null, + }; + }); +} + +function sourceStatsByRow(repoRoot, rows) { + return Object.fromEntries( + rows.map((item) => { + const files = unique(item.sourceRoots.flatMap((sourcePath) => listFiles(repoRoot, sourcePath))); + return [ + item.reportRowId, + { + sourceRootCount: item.sourceRoots.length, + sourceRootPresentCount: item.sourceRoots.filter((sourcePath) => sourceExists(repoRoot, sourcePath)).length, + sourceFileCount: files.length, + sourceFileHashes: Object.fromEntries(files.map((file) => [file, digest(readSource(repoRoot, file))])), + }, + ]; + }), + ); +} + +function buildPredicateResults(repoRoot, context) { + const promptBenchmarkRunner = readSource(repoRoot, SOURCE_ROOTS.promptBenchmarkRunner); + const promptBenchmarkTypes = readSource(repoRoot, SOURCE_ROOTS.promptBenchmarkTypes); + const readingPipelineContract = readSource(repoRoot, SOURCE_ROOTS.readingPipelineContract); + const readingObservability = readSource(repoRoot, SOURCE_ROOTS.readingObservability); + const readingOperationalTelemetry = readSource(repoRoot, SOURCE_ROOTS.readingOperationalTelemetry); + const boundedStructuredInference = readSource(repoRoot, SOURCE_ROOTS.boundedStructuredInference); + const depositorySearch = readSource(repoRoot, SOURCE_ROOTS.depositorySearch); + const embeddingConfig = readSource(repoRoot, SOURCE_ROOTS.embeddingConfig); + const readFitsRuntime = readSource(repoRoot, SOURCE_ROOTS.readFitsRuntime); + const conversationStreamEvents = readSource(repoRoot, SOURCE_ROOTS.conversationStreamEvents); + const conversationTelemetry = readSource(repoRoot, SOURCE_ROOTS.conversationTelemetry); + const pipelineLog = readSource(repoRoot, SOURCE_ROOTS.pipelineLog); + const pipelineLogHeader = readSource(repoRoot, SOURCE_ROOTS.pipelineLogHeader); + const pipelineLogTest = readSource(repoRoot, SOURCE_ROOTS.pipelineLogTest); + const pipelineLogHeaderTest = readSource(repoRoot, SOURCE_ROOTS.pipelineLogHeaderTest); + const gate3Contracts = readSource(repoRoot, SOURCE_ROOTS.gate3ContractsSource); + const spec = readSource(repoRoot, SOURCE_ROOTS.spec); + const delta = readSource(repoRoot, SOURCE_ROOTS.delta); + const notes = readSource(repoRoot, SOURCE_ROOTS.notes); + const parity = readSource(repoRoot, SOURCE_ROOTS.parity); + const roadmap = readSource(repoRoot, SOURCE_ROOTS.roadmap); + const readme = readSource(repoRoot, SOURCE_ROOTS.readme); + const protocolReadme = readSource(repoRoot, SOURCE_ROOTS.protocolReadme); + const packageJson = readSource(repoRoot, SOURCE_ROOTS.packageJson); + const gateWorkflow = readSource(repoRoot, SOURCE_ROOTS.gateWorkflow); + const canonWorkflow = readSource(repoRoot, SOURCE_ROOTS.canonWorkflow); + + return [ + predicate('gate2-inventory-passing', 'post-rewrite-prompt-inventory-delta', SOURCE_ROOTS.gate2InventoryArtifact, context.gate2Inventory.passed), + predicate('gate5-readneed-passing', 'post-rewrite-prompt-inventory-delta', SOURCE_ROOTS.gate5ReadNeedArtifact, context.gate5ReadNeed.passed), + predicate('gate6-readfitsfinding-passing', 'post-rewrite-prompt-inventory-delta', SOURCE_ROOTS.gate6ReadFitsArtifact, context.gate6ReadFits.passed), + predicate('gate7-conversation-interface-passing', 'post-rewrite-prompt-inventory-delta', SOURCE_ROOTS.gate7ConversationArtifact, context.gate7Conversation.passed), + predicate( + 'post-rewrite-artifacts-present', + 'post-rewrite-prompt-inventory-delta', + '.bitcode', + [ + SOURCE_ROOTS.gate2InventoryArtifact, + SOURCE_ROOTS.gate5ReadNeedArtifact, + SOURCE_ROOTS.gate6ReadFitsArtifact, + SOURCE_ROOTS.gate7ConversationArtifact, + ].every((sourcePath) => sourceExists(repoRoot, sourcePath)), + ), + predicate( + 'post-rewrite-report-does-not-serialize-raw-prompt-text', + 'post-rewrite-prompt-inventory-delta', + 'sourceSafety', + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ROWS.every((item) => + item.rawPromptTextSerialized === false + && item.rawInterpolatedPromptSerialized === false + && item.rawProviderResponseSerialized === false, + ), + ), + predicate('gate4-reading-benchmark-passing', 'readneed-post-rewrite-benchmark-delta', SOURCE_ROOTS.gate4BaselinesArtifact, context.gate4Baselines.passed), + predicate('readneed-hardening-row-count', 'readneed-post-rewrite-benchmark-delta', SOURCE_ROOTS.gate5ReadNeedArtifact, context.gate5ReadNeed.coverage?.rowCount >= 7), + predicate( + 'readneed-benchmark-baselines-have-quality-scores', + 'readneed-post-rewrite-benchmark-delta', + SOURCE_ROOTS.gate4BaselinesArtifact, + context.gate4Baselines.coverage?.baselineScoreMinimum >= 0.8 + && context.gate4Baselines.coverage?.readNeedBaselineRowCount >= 3, + ), + predicate( + 'readneed-observability-projects-parsed-output', + 'readneed-post-rewrite-benchmark-delta', + SOURCE_ROOTS.readingObservability, + readingObservability.includes('parsedTypedOutputPresent') && readingObservability.includes('outputSchema'), + ), + predicate( + 'readneed-source-safe-metadata-only', + 'readneed-post-rewrite-benchmark-delta', + 'sourceSafety', + context.gate5ReadNeed.sourceSafety?.sourceSafeMetadataOnly === true + && context.gate5ReadNeed.sourceSafety?.rawPromptTextSerialized === false + && context.gate5ReadNeed.sourceSafety?.rawProviderResponseSerialized === false, + ), + predicate('readfitsfinding-hardening-passing', 'readfitsfinding-post-rewrite-benchmark-delta', SOURCE_ROOTS.gate6ReadFitsArtifact, context.gate6ReadFits.passed), + predicate( + 'readfitsfinding-has-many-candidate-search', + 'readfitsfinding-post-rewrite-benchmark-delta', + SOURCE_ROOTS.depositorySearch, + /candidate|rank|threshold/iu.test(depositorySearch) && readFitsRuntime.includes('candidate'), + ), + predicate( + 'readfitsfinding-embedding-config-present', + 'readfitsfinding-post-rewrite-benchmark-delta', + SOURCE_ROOTS.embeddingConfig, + /embedding/iu.test(embeddingConfig) && /vector|model|dimension/iu.test(embeddingConfig), + ), + predicate( + 'readfitsfinding-search-embeddings-passing', + 'readfitsfinding-post-rewrite-benchmark-delta', + SOURCE_ROOTS.v38ReadFitsSearchEmbeddingsSource, + context.v38ReadFitsSearchEmbeddings.passed, + ), + predicate( + 'readfitsfinding-source-safe-preview-only', + 'readfitsfinding-post-rewrite-benchmark-delta', + 'sourceSafety', + context.gate6ReadFits.sourceSafety?.sourceSafeMetadataOnly === true + && context.gate6ReadFits.sourceSafety?.unpaidAssetPackSourceVisible === false, + ), + predicate('gate7-conversation-artifact-passing', 'conversation-interface-post-rewrite-benchmark-delta', SOURCE_ROOTS.gate7ConversationArtifact, context.gate7Conversation.passed), + predicate( + 'conversation-stream-template-id-only', + 'conversation-interface-post-rewrite-benchmark-delta', + SOURCE_ROOTS.conversationStreamEvents, + conversationStreamEvents.includes("promptDisclosurePosture: 'prompt_template_id_only'"), + ), + predicate( + 'conversation-telemetry-redacts-protected-context', + 'conversation-interface-post-rewrite-benchmark-delta', + SOURCE_ROOTS.conversationTelemetry, + conversationTelemetry.includes('PROTECTED_CONTEXT_RE') && conversationTelemetry.includes('SECRET_VALUE_PATTERNS'), + ), + predicate( + 'pipeline-log-projects-prompt-result-disclosure', + 'conversation-interface-post-rewrite-benchmark-delta', + SOURCE_ROOTS.pipelineLog, + pipelineLog.includes('promptDisclosurePosture') && pipelineLog.includes('resultDisclosurePosture'), + ), + predicate( + 'conversation-interface-artifact-source-safe', + 'conversation-interface-post-rewrite-benchmark-delta', + 'sourceSafety', + context.gate7Conversation.sourceSafety?.sourceSafeMetadataOnly === true + && context.gate7Conversation.sourceSafety?.rawPromptTextSerialized === false + && context.gate7Conversation.sourceSafety?.rawProviderResponseSerialized === false, + ), + predicate('gate3-registry-contracts-passing', 'registry-lineage-version-telemetry-binding', SOURCE_ROOTS.gate3ContractsArtifact, context.gate3Contracts.passed), + predicate( + 'benchmark-types-have-pbversion-and-quality-gates', + 'registry-lineage-version-telemetry-binding', + SOURCE_ROOTS.promptBenchmarkTypes, + promptBenchmarkTypes.includes('PBVersion') && promptBenchmarkTypes.includes('QualityGates') && promptBenchmarkTypes.includes('PromptBenchmarkResult'), + ), + predicate( + 'benchmark-runner-handles-promptpart-and-prompt', + 'registry-lineage-version-telemetry-binding', + SOURCE_ROOTS.promptBenchmarkRunner, + promptBenchmarkRunner.includes('benchmarkPromptPart') && promptBenchmarkRunner.includes('benchmarkPrompt('), + ), + predicate( + 'registry-contracts-bind-hierarchy', + 'registry-lineage-version-telemetry-binding', + SOURCE_ROOTS.gate3ContractsSource, + gate3Contracts.includes('registry-lineage') || (gate3Contracts.includes('hierarchy') && gate3Contracts.includes('interpolation')), + ), + predicate( + 'registry-lineage-source-safe', + 'registry-lineage-version-telemetry-binding', + 'sourceSafety', + context.gate3Contracts.sourceSafety?.sourceSafeMetadataOnly === true + && context.v38PromptBenchmark.coverage?.sourceSafeMetadataOnly === true, + ), + predicate('v38-ptrr-stack-passing', 'failsafe-thricified-inference-receipt-projection', SOURCE_ROOTS.v38PtrrStackSource, context.v38PtrrStack.passed), + predicate('v38-inference-telemetry-passing', 'failsafe-thricified-inference-receipt-projection', SOURCE_ROOTS.v38InferenceTelemetryArtifact, context.v38InferenceTelemetry.passed), + predicate( + 'inference-telemetry-has-required-levels', + 'failsafe-thricified-inference-receipt-projection', + SOURCE_ROOTS.v38InferenceTelemetrySource, + ['failsafe', 'thricified_generation', 'interpolated_prompt', 'raw_response', 'parsed_output', 'schema_verdict'].every((levelId) => + context.v38InferenceTelemetry.telemetryLevelIds.includes(levelId), + ), + ), + predicate( + 'bounded-inference-records-raw-and-parsed', + 'failsafe-thricified-inference-receipt-projection', + SOURCE_ROOTS.boundedStructuredInference, + boundedStructuredInference.includes('rawResponse') && boundedStructuredInference.includes('parsedTypedOutput'), + ), + predicate( + 'observability-projects-generation-receipts', + 'failsafe-thricified-inference-receipt-projection', + SOURCE_ROOTS.readingObservability, + readingObservability.includes('thricifiedGenerationId') && readingObservability.includes('generationPromptIds'), + ), + predicate( + 'failsafe-thricified-source-safe', + 'failsafe-thricified-inference-receipt-projection', + 'sourceSafety', + context.v38InferenceTelemetry.coverage?.sourceSafeMetadataOnly === true + && context.v38InferenceTelemetry.coverage?.rawProviderResponseVisible === false, + ), + predicate('reading-operational-telemetry-passing', 'rich-stream-source-safe-telemetry-projection', SOURCE_ROOTS.v39OperationalTelemetryArtifact, context.v39OperationalTelemetry.passed), + predicate( + 'stream-events-sanitize-metadata', + 'rich-stream-source-safe-telemetry-projection', + SOURCE_ROOTS.conversationStreamEvents, + conversationStreamEvents.includes('sanitizeMetadata') && conversationStreamEvents.includes('rawPrompt') && conversationStreamEvents.includes('rawResponse'), + ), + predicate( + 'rich-log-header-renders-prompt-result-posture', + 'rich-stream-source-safe-telemetry-projection', + SOURCE_ROOTS.pipelineLogHeader, + pipelineLogHeader.includes('promptDisclosurePosture') && pipelineLogHeader.includes('resultDisclosurePosture') && pipelineLogHeader.includes('failClosedState'), + ), + predicate( + 'rich-log-tests-cover-telemetry-disclosure', + 'rich-stream-source-safe-telemetry-projection', + `${SOURCE_ROOTS.pipelineLogTest},${SOURCE_ROOTS.pipelineLogHeaderTest}`, + pipelineLogTest.includes('prompt_template_id_only') && pipelineLogHeaderTest.includes('parsed_result_shape'), + ), + predicate( + 'rich-stream-source-safe-only', + 'rich-stream-source-safe-telemetry-projection', + 'sourceSafety', + context.v39OperationalTelemetry.coverage?.sourceSafeMetadataOnly === true + && context.gate7Conversation.sourceSafety?.sourceSafeMetadataOnly === true, + ), + predicate( + 'repair-hooks-present', + 'repair-hooks-parsed-output-redaction-posture', + SOURCE_ROOTS.readingOperationalTelemetry, + readingOperationalTelemetry.includes('ReadingOperationalRepairRunbookHook') && readingOperationalTelemetry.includes('repairActions'), + ), + predicate( + 'repair-readback-redacts-raw-prompts', + 'repair-hooks-parsed-output-redaction-posture', + SOURCE_ROOTS.readingOperationalTelemetry, + readingOperationalTelemetry.includes('rawProtectedPromptVisible: false') + && readingOperationalTelemetry.includes('rawProviderResponseVisible: false') + && readingOperationalTelemetry.includes('rawInterpolatedPromptVisible: false'), + ), + predicate( + 'conversation-telemetry-redacts-secret-values', + 'repair-hooks-parsed-output-redaction-posture', + SOURCE_ROOTS.conversationTelemetry, + conversationTelemetry.includes('redactPemPrivateKeyBlocks') && conversationTelemetry.includes('SECRET_VALUE_PATTERNS'), + ), + predicate( + 'parsed-output-shape-source-safe', + 'repair-hooks-parsed-output-redaction-posture', + SOURCE_ROOTS.readingObservability, + readingObservability.includes('parsedTypedOutputPresent') && readingObservability.includes('outputSchema'), + ), + predicate( + 'repair-hook-artifacts-source-safe', + 'repair-hooks-parsed-output-redaction-posture', + 'sourceSafety', + context.v39OperationalTelemetry.coverage?.rawProviderResponseVisible === false + && context.v39OperationalTelemetry.coverage?.rawInterpolatedPromptVisible === false + && context.v39OperationalTelemetry.coverage?.credentialsSerialized === false, + ), + predicate('gate8-source-exists', 'gate8-tests-docs-workflows', SOURCE_ROOTS.packageSource, sourceExists(repoRoot, SOURCE_ROOTS.packageSource)), + predicate('gate8-test-exists', 'gate8-tests-docs-workflows', SOURCE_ROOTS.packageTest, sourceExists(repoRoot, SOURCE_ROOTS.packageTest)), + predicate('gate8-generator-exists', 'gate8-tests-docs-workflows', SOURCE_ROOTS.generator, sourceExists(repoRoot, SOURCE_ROOTS.generator)), + predicate('gate8-checker-exists', 'gate8-tests-docs-workflows', SOURCE_ROOTS.checker, sourceExists(repoRoot, SOURCE_ROOTS.checker)), + predicate( + 'package-json-exposes-gate8-scripts', + 'gate8-tests-docs-workflows', + SOURCE_ROOTS.packageJson, + packageJson.includes('generate:v41-prompt-program-benchmark-report') + && packageJson.includes('check:v41-prompt-program-benchmark-report') + && packageJson.includes('check:v41-gate8'), + ), + predicate( + 'workflows-run-gate8-checker', + 'gate8-tests-docs-workflows', + `${SOURCE_ROOTS.gateWorkflow},${SOURCE_ROOTS.canonWorkflow}`, + gateWorkflow.includes('check-v41-gate8-prompt-program-benchmark-report.mjs') + && canonWorkflow.includes('check-v41-gate8-prompt-program-benchmark-report.mjs'), + ), + predicate( + 'spec-documents-gate8-closure', + 'gate8-tests-docs-workflows', + SOURCE_ROOTS.spec, + spec.includes('V41 Gate 8 Prompt Benchmark Report And Telemetry Integration') + && spec.includes(V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ARTIFACT_PATH) + && delta.includes('Gate 8') + && notes.includes('Gate 8 implementation notes') + && parity.includes('Prompt benchmark report and telemetry artifact'), + ), + predicate( + 'roadmap-documents-gate8-closure', + 'gate8-tests-docs-workflows', + SOURCE_ROOTS.roadmap, + roadmap.includes('V41 Gate 8 closure anchor') + && roadmap.includes('Current working gate: V41 Gate 8') + && roadmap.includes('Next queued gate after V41 Gate 8'), + ), + predicate( + 'readmes-document-gate8-helpers', + 'gate8-tests-docs-workflows', + `${SOURCE_ROOTS.readme},${SOURCE_ROOTS.protocolReadme}`, + readme.includes('V41 Gate 8') && protocolReadme.includes('V41PromptProgramBenchmarkReport'), + ), + ]; +} + +function enrichRows(rows, sourceStats, predicateResults) { + return rows.map((item) => { + const rowPredicates = predicateResults.filter((result) => result.rowId === item.reportRowId); + return { + ...item, + sourceStats: sourceStats[item.reportRowId], + requiredPredicateCount: rowPredicates.length, + passedPredicateCount: rowPredicates.filter((result) => result.passed).length, + failedPredicateIds: rowPredicates.filter((result) => !result.passed).map((result) => result.id), + passed: rowPredicates.length === item.requiredPredicateIds.length + && rowPredicates.every((result) => result.passed) + && sourceStats[item.reportRowId].sourceRootPresentCount === sourceStats[item.reportRowId].sourceRootCount, + }; + }); +} + +function dependencyRoots(context) { + return { + v38PromptBenchmarkReportRoot: context.v38PromptBenchmark.artifactRoot, + v38InferenceTelemetryDisclosureRoot: context.v38InferenceTelemetry.artifactRoot, + v38PtrrFailsafeThricifiedStackRoot: context.v38PtrrStack.artifactRoot, + v38ReadFitsFindingSearchEmbeddingsRoot: context.v38ReadFitsSearchEmbeddings.artifactRoot, + v39OperationalTelemetryRepairReadbackRoot: context.v39OperationalTelemetry.artifactRoot, + v40PromptBenchmarkSmokeRoot: context.v40PromptBenchmarkSmoke.artifactRoot, + gate2InventoryRoot: context.gate2Inventory.artifactRoot, + gate3RegistryInterpolationRoot: context.gate3Contracts.artifactRoot, + gate4ReadingPromptBenchmarkBaselineRoot: context.gate4Baselines.artifactRoot, + gate5ReadNeedPromptHardeningRoot: context.gate5ReadNeed.artifactRoot, + gate6ReadFitsFindingPromptHardeningRoot: context.gate6ReadFits.artifactRoot, + gate7ConversationToolInterfacePromptRewriteRoot: context.gate7Conversation.artifactRoot, + }; +} + +export function buildV41PromptProgramBenchmarkReport(input = {}) { + const generatedAt = typeof input.generatedAt === 'string' ? input.generatedAt : '2026-05-26T00:00:00.000Z'; + const repoRoot = typeof input.repoRoot === 'string' ? input.repoRoot : DEFAULT_REPO_ROOT; + const context = dependencyContext(generatedAt, repoRoot); + const sourceEvidence = buildSourceEvidence(repoRoot, V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ROWS); + const sourceStats = sourceStatsByRow(repoRoot, V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ROWS); + const sourcePredicateResults = buildPredicateResults(repoRoot, context); + const rows = enrichRows(V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ROWS, sourceStats, sourcePredicateResults); + const failedPredicateIds = sourcePredicateResults.filter((result) => !result.passed).map((result) => result.id); + const missingSourceRoots = sourceEvidence.filter((entry) => !entry.exists).map((entry) => entry.sourcePath); + const legacySourceRoots = sourceEvidence.filter((entry) => entry.legacy).map((entry) => entry.sourcePath); + const promptProgramArtifactIds = unique(rows.flatMap((item) => item.promptProgramArtifactIds)); + const metricIds = unique(rows.flatMap((item) => item.metricIds)); + const telemetryReceiptIds = unique(rows.flatMap((item) => item.telemetryReceiptIds)); + const dependencyRootValues = dependencyRoots(context); + + const dependenciesPassing = Object.entries(context).every(([, artifact]) => artifact.passed === true); + const sourceSafeMetadataOnly = rows.every((item) => + item.sourceSafeMetadataOnly === true + && item.rawPromptTextSerialized === false + && item.rawInterpolatedPromptSerialized === false + && item.rawProviderResponseSerialized === false + && item.protectedPromptSerialized === false + && item.protectedSourceVisible === false + && item.privateContextSerialized === false + && item.credentialsSerialized === false + && item.unpaidAssetPackSourceVisible === false + && item.walletPrivateMaterialVisible === false + && item.settlementPrivatePayloadVisible === false, + ); + + const failures = []; + if (!dependenciesPassing) failures.push('One or more dependency prompt benchmark or telemetry artifacts are not passing.'); + if (failedPredicateIds.length) failures.push(`failed Gate 8 predicates: ${failedPredicateIds.join(', ')}`); + if (missingSourceRoots.length) failures.push(`missing source roots: ${missingSourceRoots.join(', ')}`); + if (legacySourceRoots.length) failures.push(`legacy source roots present: ${legacySourceRoots.join(', ')}`); + if (!sourceSafeMetadataOnly) failures.push('Gate 8 rows are not source-safe metadata only.'); + + const coverage = { + rowCount: rows.length, + metricIds, + metricCount: metricIds.length, + promptProgramArtifactIds, + promptProgramArtifactCount: promptProgramArtifactIds.length, + telemetryReceiptIds, + telemetryReceiptCount: telemetryReceiptIds.length, + sourceRootCount: sourceEvidence.length, + sourceRootPresentCount: sourceEvidence.filter((entry) => entry.exists).length, + missingSourceRoots, + legacySourceRoots: legacySourceRoots.length > 0, + requiredPredicateCount: sourcePredicateResults.length, + passedPredicateCount: sourcePredicateResults.filter((result) => result.passed).length, + failedPredicateIds, + passingRowCount: rows.filter((item) => item.passed).length, + failingRowIds: rows.filter((item) => !item.passed).map((item) => item.reportRowId), + sourceSafeMetadataOnly, + dependenciesPassing, + dependencyRoots: dependencyRootValues, + v38PromptBenchmarkRowCount: context.v38PromptBenchmark.coverage?.rowCount ?? 0, + v38InferenceTelemetryLevelCount: context.v38InferenceTelemetry.coverage?.requiredTelemetryLevelCount ?? 0, + v41PromptInventoryRowCount: context.gate2Inventory.coverage?.rowCount ?? 0, + v41ReadingBenchmarkBaselineRowCount: context.gate4Baselines.coverage?.rowCount ?? 0, + v41ReadNeedHardeningRowCount: context.gate5ReadNeed.coverage?.rowCount ?? 0, + v41ReadFitsFindingHardeningRowCount: context.gate6ReadFits.coverage?.rowCount ?? 0, + v41ConversationToolInterfaceRowCount: context.gate7Conversation.coverage?.rowCount ?? 0, + }; + + const sourceSafety = { + sourceSafetyClass: 'source_safe_prompt_program_benchmark_telemetry_metadata', + sourceSafeMetadataOnly, + rawPromptTextSerialized: false, + rawInterpolatedPromptSerialized: false, + rawProviderResponseSerialized: false, + protectedPromptSerialized: false, + protectedSourceVisible: false, + privateContextSerialized: false, + credentialsSerialized: false, + unpaidAssetPackSourceVisible: false, + walletPrivateMaterialVisible: false, + settlementPrivatePayloadVisible: false, + forbiddenPayloadClasses: [...FORBIDDEN_PAYLOAD_CLASSES], + }; + + const root = artifactRoot( + JSON.stringify({ + rows: rows.map((item) => item.rowRoot), + metrics: metricIds, + receipts: telemetryReceiptIds, + predicates: sourcePredicateResults.filter((result) => result.passed).map((result) => result.id), + roots: dependencyRootValues, + }), + ); + + return { + artifactId: 'v41-prompt-program-benchmark-report', + schemaId: V41_PROMPT_PROGRAM_BENCHMARK_REPORT_SCHEMA_ID, + version: V41_PROMPT_PROGRAM_BENCHMARK_REPORT_VERSION, + currentTarget: V41_PROMPT_PROGRAM_BENCHMARK_REPORT_CURRENT_TARGET, + generatedAt, + artifactPath: V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ARTIFACT_PATH, + artifactRoot: root, + sourceSafetyVerdict: V41_PROMPT_PROGRAM_BENCHMARK_REPORT_SOURCE_SAFETY_VERDICT, + metricIds: [...V41_PROMPT_PROGRAM_BENCHMARK_REPORT_METRIC_IDS], + disclosureTiers: [...V41_PROMPT_PROGRAM_BENCHMARK_REPORT_DISCLOSURE_TIERS], + rows, + sourceEvidence, + sourceStatsByRow: sourceStats, + sourcePredicateResults, + dependencyRoots: dependencyRootValues, + sourceSafety, + coverage, + failures, + passed: + failures.length === 0 + && coverage.rowCount === 9 + && coverage.metricCount === V41_PROMPT_PROGRAM_BENCHMARK_REPORT_METRIC_IDS.length + && coverage.promptProgramArtifactCount >= 10 + && coverage.telemetryReceiptCount >= 20 + && coverage.sourceRootPresentCount === coverage.sourceRootCount + && coverage.passedPredicateCount === coverage.requiredPredicateCount + && coverage.failingRowIds.length === 0 + && coverage.dependenciesPassing + && coverage.sourceSafeMetadataOnly + && coverage.legacySourceRoots === false + && coverage.v38PromptBenchmarkRowCount >= 7 + && coverage.v38InferenceTelemetryLevelCount >= 13 + && coverage.v41ReadingBenchmarkBaselineRowCount >= 10 + && coverage.v41ReadNeedHardeningRowCount >= 7 + && coverage.v41ReadFitsFindingHardeningRowCount >= 8 + && coverage.v41ConversationToolInterfaceRowCount >= 9, + }; +} diff --git a/packages/protocol/src/canonical/v41-readfitsfinding-prompt-hardening.js b/packages/protocol/src/canonical/v41-readfitsfinding-prompt-hardening.js index b4a2d279c..6fcdb40d6 100644 --- a/packages/protocol/src/canonical/v41-readfitsfinding-prompt-hardening.js +++ b/packages/protocol/src/canonical/v41-readfitsfinding-prompt-hardening.js @@ -578,13 +578,13 @@ function predicatesForRow(repoRoot, rowData) { 'docs.current-gate-6', rowId, SOURCE_ROOTS.roadmap, - /Current working gate: V41 Gate (?:6|7)/u.test(source), + /Current working gate: V41 Gate (?:6|7|8|9)/u.test(source), ), predicate( 'docs.next-gate-7', rowId, SOURCE_ROOTS.roadmap, - /Next queued gate after V41 Gate (?:6|7): V41 (?:Conversation Tool And Interface Prompt Rewrite|Prompt Benchmark Report And Telemetry Integration)/u.test(source), + /Next queued gate after V41 Gate (?:6|7|8): V41 (?:Conversation Tool And Interface Prompt Rewrite|Prompt Benchmark Report And Telemetry Integration|Promotion Readiness)/u.test(source), ), predicate('docs.gate6-closure-anchor', rowId, SOURCE_ROOTS.roadmap, /V41 Gate 6 closure anchor/u.test(source)), predicate('parity.finding-fits-implemented', rowId, SOURCE_ROOTS.parity, /Finding Fits rewrite.+implemented/us.test(source)), diff --git a/packages/protocol/src/index.d.ts b/packages/protocol/src/index.d.ts index ba44ef2c7..7ef67a505 100644 --- a/packages/protocol/src/index.d.ts +++ b/packages/protocol/src/index.d.ts @@ -494,6 +494,15 @@ export const V41_CONVERSATION_TOOL_INTERFACE_PROMPT_REWRITE_METRIC_IDS: readonly export const V41_CONVERSATION_TOOL_INTERFACE_PROMPT_REWRITE_DISCLOSURE_TIERS: readonly string[]; export const V41_CONVERSATION_TOOL_INTERFACE_PROMPT_REWRITE_ROWS: readonly Record[]; export function buildV41ConversationToolInterfacePromptRewrite(input?: Record): BitcodeProtocolReport; +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ARTIFACT_PATH: string; +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_CURRENT_TARGET: string; +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_SCHEMA_ID: string; +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_VERSION: string; +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_SOURCE_SAFETY_VERDICT: string; +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_METRIC_IDS: readonly string[]; +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_DISCLOSURE_TIERS: readonly string[]; +export const V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ROWS: readonly Record[]; +export function buildV41PromptProgramBenchmarkReport(input?: Record): BitcodeProtocolReport; export const EXCHANGE_INTENT_ORDER_CONTRACTS_ARTIFACT_PATH: string; export const EXCHANGE_INTENT_ORDER_CONTRACTS_CURRENT_TARGET: string; export const EXCHANGE_INTENT_ORDER_CONTRACTS_SCHEMA_ID: string; diff --git a/packages/protocol/src/index.js b/packages/protocol/src/index.js index 080c1ba5d..8c087e7fb 100644 --- a/packages/protocol/src/index.js +++ b/packages/protocol/src/index.js @@ -548,6 +548,17 @@ export { V41_CONVERSATION_TOOL_INTERFACE_PROMPT_REWRITE_ROWS, buildV41ConversationToolInterfacePromptRewrite } from './canonical/v41-conversation-tool-interface-prompt-rewrite.js'; +export { + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ARTIFACT_PATH, + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_CURRENT_TARGET, + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_SCHEMA_ID, + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_SOURCE_SAFETY_VERDICT, + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_VERSION, + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_DISCLOSURE_TIERS, + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_METRIC_IDS, + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ROWS, + buildV41PromptProgramBenchmarkReport +} from './canonical/v41-prompt-program-benchmark-report.js'; export { EXCHANGE_INTENT_ACTION_KINDS, EXCHANGE_INTENT_ORDER_CONTRACTS_ARTIFACT_PATH, diff --git a/packages/protocol/test/v41-prompt-program-benchmark-report.test.js b/packages/protocol/test/v41-prompt-program-benchmark-report.test.js new file mode 100644 index 000000000..3ef6eb5c8 --- /dev/null +++ b/packages/protocol/test/v41-prompt-program-benchmark-report.test.js @@ -0,0 +1,122 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { test } from 'node:test'; + +import { + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ARTIFACT_PATH, + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_DISCLOSURE_TIERS, + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_METRIC_IDS, + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ROWS, + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_SOURCE_SAFETY_VERDICT, + buildV41PromptProgramBenchmarkReport, +} from '../src/index.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(__dirname, '..', '..', '..'); + +test('builds source-safe V41 prompt-program benchmark report artifact', () => { + const artifact = buildV41PromptProgramBenchmarkReport({ + generatedAt: '2026-05-26T00:00:00.000Z', + repoRoot, + }); + + assert.equal(artifact.artifactId, 'v41-prompt-program-benchmark-report'); + assert.equal(artifact.schemaId, 'bitcode.v41.promptProgramBenchmarkReport.v1'); + assert.equal(artifact.version, 'V41'); + assert.equal(artifact.currentTarget, 'V40'); + assert.equal(artifact.passed, true); + assert.equal(artifact.artifactPath, V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ARTIFACT_PATH); + assert.equal(artifact.sourceSafetyVerdict, V41_PROMPT_PROGRAM_BENCHMARK_REPORT_SOURCE_SAFETY_VERDICT); + assert.match(artifact.artifactRoot, /^v41-prompt-program-benchmark-report:[a-f0-9]{24}$/u); + assert.deepEqual(artifact.metricIds, [...V41_PROMPT_PROGRAM_BENCHMARK_REPORT_METRIC_IDS]); + assert.deepEqual(artifact.disclosureTiers, [...V41_PROMPT_PROGRAM_BENCHMARK_REPORT_DISCLOSURE_TIERS]); + assert.equal(artifact.rows.length, V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ROWS.length); +}); + +test('binds V38, V40, and V41 prompt benchmark and telemetry dependencies', () => { + const artifact = buildV41PromptProgramBenchmarkReport({ + generatedAt: '2026-05-26T00:00:00.000Z', + repoRoot, + }); + + assert.equal(artifact.coverage.rowCount, 9); + assert.equal(artifact.coverage.metricCount, V41_PROMPT_PROGRAM_BENCHMARK_REPORT_METRIC_IDS.length); + assert.equal(artifact.coverage.promptProgramArtifactCount >= 10, true); + assert.equal(artifact.coverage.telemetryReceiptCount >= 20, true); + assert.equal(artifact.coverage.sourceRootPresentCount, artifact.coverage.sourceRootCount); + assert.equal(artifact.coverage.passedPredicateCount, artifact.coverage.requiredPredicateCount); + assert.deepEqual(artifact.coverage.failedPredicateIds, []); + assert.deepEqual(artifact.coverage.failingRowIds, []); + assert.equal(artifact.coverage.dependenciesPassing, true); + assert.equal(artifact.coverage.v38PromptBenchmarkRowCount >= 7, true); + assert.equal(artifact.coverage.v38InferenceTelemetryLevelCount >= 13, true); + assert.equal(artifact.coverage.v41ReadingBenchmarkBaselineRowCount >= 10, true); + assert.equal(artifact.coverage.v41ReadNeedHardeningRowCount >= 7, true); + assert.equal(artifact.coverage.v41ReadFitsFindingHardeningRowCount >= 8, true); + assert.equal(artifact.coverage.v41ConversationToolInterfaceRowCount >= 9, true); +}); + +test('keeps benchmark telemetry disclosure source-safe', () => { + const artifact = buildV41PromptProgramBenchmarkReport({ + generatedAt: '2026-05-26T00:00:00.000Z', + repoRoot, + }); + + assert.match(artifact.dependencyRoots.v38PromptBenchmarkReportRoot, /^v38-prompt-benchmark-report:[a-f0-9]{24}$/u); + assert.match( + artifact.dependencyRoots.v38InferenceTelemetryDisclosureRoot, + /^v38-inference-telemetry-disclosure-report:[a-f0-9]{24}$/u, + ); + assert.match(artifact.dependencyRoots.v38PtrrFailsafeThricifiedStackRoot, /^v38-ptrr-failsafe-thricified-stack:[a-f0-9]{24}$/u); + assert.match( + artifact.dependencyRoots.v38ReadFitsFindingSearchEmbeddingsRoot, + /^v38-read-fits-finding-search:[a-f0-9]{24}$/u, + ); + assert.match( + artifact.dependencyRoots.v39OperationalTelemetryRepairReadbackRoot, + /^v39-operational-telemetry-repair-readback:[a-f0-9]{24}$/u, + ); + assert.match(artifact.dependencyRoots.v40PromptBenchmarkSmokeRoot, /^v40-prompt-benchmark-smoke-v41-readiness:[a-f0-9]{24}$/u); + assert.match(artifact.dependencyRoots.gate2InventoryRoot, /^v41-promptpart-prompt-inventory:[a-f0-9]{24}$/u); + assert.match(artifact.dependencyRoots.gate3RegistryInterpolationRoot, /^v41-registry-interpolation-contract:[a-f0-9]{24}$/u); + assert.match( + artifact.dependencyRoots.gate4ReadingPromptBenchmarkBaselineRoot, + /^v41-reading-prompt-benchmark-baselines:[a-f0-9]{24}$/u, + ); + assert.match(artifact.dependencyRoots.gate5ReadNeedPromptHardeningRoot, /^v41-readneed-prompt-hardening:[a-f0-9]{24}$/u); + assert.match(artifact.dependencyRoots.gate6ReadFitsFindingPromptHardeningRoot, /^v41-readfitsfinding-prompt-hardening:[a-f0-9]{24}$/u); + assert.match( + artifact.dependencyRoots.gate7ConversationToolInterfacePromptRewriteRoot, + /^v41-conversation-tool-interface-prompt-rewrite:[a-f0-9]{24}$/u, + ); + + assert.equal(artifact.sourceSafety.sourceSafeMetadataOnly, true); + assert.equal(artifact.sourceSafety.rawPromptTextSerialized, false); + assert.equal(artifact.sourceSafety.rawInterpolatedPromptSerialized, false); + assert.equal(artifact.sourceSafety.rawProviderResponseSerialized, false); + assert.equal(artifact.sourceSafety.protectedPromptSerialized, false); + assert.equal(artifact.sourceSafety.protectedSourceVisible, false); + assert.equal(artifact.sourceSafety.privateContextSerialized, false); + assert.equal(artifact.sourceSafety.unpaidAssetPackSourceVisible, false); + assert.equal(artifact.sourceSafety.credentialsSerialized, false); + assert.equal(artifact.sourceSafety.walletPrivateMaterialVisible, false); + assert.equal(artifact.sourceSafety.settlementPrivatePayloadVisible, false); + + for (const row of artifact.rows) { + assert.match(row.rowRoot, /^v41-prompt-program-benchmark-report-row:[a-f0-9]{24}$/u); + assert.equal(row.sourceSafetyClass, 'source_safe_prompt_program_benchmark_telemetry_metadata'); + assert.equal(row.sourceSafeMetadataOnly, true); + assert.equal(row.rawPromptTextSerialized, false); + assert.equal(row.rawInterpolatedPromptSerialized, false); + assert.equal(row.rawProviderResponseSerialized, false); + assert.equal(row.protectedPromptSerialized, false); + assert.equal(row.protectedSourceVisible, false); + assert.equal(row.privateContextSerialized, false); + assert.equal(row.unpaidAssetPackSourceVisible, false); + assert.equal(row.credentialsSerialized, false); + assert.equal(row.walletPrivateMaterialVisible, false); + assert.equal(row.settlementPrivatePayloadVisible, false); + assert.equal(row.passed, true); + } +}); diff --git a/scripts/check-v41-gate1-prompt-program-roadmap-opening.mjs b/scripts/check-v41-gate1-prompt-program-roadmap-opening.mjs index 529dfdce9..158e8d022 100644 --- a/scripts/check-v41-gate1-prompt-program-roadmap-opening.mjs +++ b/scripts/check-v41-gate1-prompt-program-roadmap-opening.mjs @@ -144,7 +144,7 @@ function main() { } assertCheck(failures, roadmap.includes('Current active canonical pointer: `BITCODE_SPEC.txt` -> `V40`'), 'Roadmap must state V40 active pointer.'); - assertCheck(failures, /Current working gate: V41 Gate (?:1|2|3|4|5|6|7)\b/u.test(roadmap), 'Roadmap must state active V41 gate progression.'); + assertCheck(failures, /Current working gate: V41 Gate (?:1|2|3|4|5|6|7|8|9)\b/u.test(roadmap), 'Roadmap must state active V41 gate progression.'); assertCheck(failures, roadmap.includes('| V41 | `BITCODE_SPEC_V41.md` | active draft target |'), 'Roadmap must list V41 as active draft target.'); assertCheck(failures, readme.includes('resolves to `V40`; V41 is the active draft target'), 'README must state V40 active / V41 draft posture.'); assertCheck(failures, protocolReadme.includes('V41 Gate 1') && protocolReadme.includes('V40` active, `V41` draft'), 'Protocol README must document V41 Gate 1 active/draft posture.'); diff --git a/scripts/check-v41-gate4-reading-prompt-benchmark-baselines.mjs b/scripts/check-v41-gate4-reading-prompt-benchmark-baselines.mjs index 61ba8c507..f04b0bdf4 100644 --- a/scripts/check-v41-gate4-reading-prompt-benchmark-baselines.mjs +++ b/scripts/check-v41-gate4-reading-prompt-benchmark-baselines.mjs @@ -270,14 +270,15 @@ function main() { assertCheck(failures, packageJson.includes('check:v41-gate4'), 'package.json must expose check:v41-gate4.'); const roadmap = fileExists(root, 'SPECIFICATIONS_ROADMAP.md') ? read(root, 'SPECIFICATIONS_ROADMAP.md') : ''; - assertCheck(failures, /Current working gate: V41 Gate (?:4|5|6|7)\b/u.test(roadmap), 'Roadmap must name V41 Gate 4 or later as current working gate.'); + assertCheck(failures, /Current working gate: V41 Gate (?:4|5|6|7|8|9)\b/u.test(roadmap), 'Roadmap must name V41 Gate 4 or later as current working gate.'); assertCheck( failures, - roadmap.includes('V41 Gate 7 closure anchor: prompt-program work now owns package-backed `V41ConversationToolInterfacePromptRewrite`') || + roadmap.includes('V41 Gate 8 closure anchor: prompt-program work now owns package-backed `V41PromptProgramBenchmarkReport`') || + roadmap.includes('V41 Gate 7 closure anchor: prompt-program work now owns package-backed `V41ConversationToolInterfacePromptRewrite`') || roadmap.includes('V41 Gate 6 closure anchor: prompt-program work now owns package-backed `V41ReadFitsFindingPromptHardening`') || roadmap.includes('V41 Gate 5 closure anchor: prompt-program work now owns package-backed `V41ReadNeedPromptHardening`') || roadmap.includes('Next queued gate after V41 Gate 4: V41 ReadNeedComprehensionSynthesis Prompt Rewrite And Return-Type Hardening.'), - 'Roadmap must preserve V41 Gate 5, Gate 6, or Gate 7 progression.', + 'Roadmap must preserve V41 Gate 5, Gate 6, Gate 7, or Gate 8 progression.', ); assertCheck(failures, roadmap.includes('V42 should focus on the reliable MVP product experience'), 'Roadmap must preserve V42 reliable MVP product note.'); assertCheck(failures, roadmap.includes('AI-reading dominant demonstration'), 'Roadmap must preserve AI-reading dominant demonstration note.'); diff --git a/scripts/check-v41-gate5-readneed-prompt-hardening.mjs b/scripts/check-v41-gate5-readneed-prompt-hardening.mjs index 0830f4fc5..cea83513e 100644 --- a/scripts/check-v41-gate5-readneed-prompt-hardening.mjs +++ b/scripts/check-v41-gate5-readneed-prompt-hardening.mjs @@ -238,15 +238,18 @@ function main() { failures, roadmap.includes('Current working gate: V41 Gate 5') || roadmap.includes('Current working gate: V41 Gate 6') || - roadmap.includes('Current working gate: V41 Gate 7'), - 'Roadmap must name V41 Gate 5, Gate 6, or Gate 7 as current working gate.', + roadmap.includes('Current working gate: V41 Gate 7') || + roadmap.includes('Current working gate: V41 Gate 8') || + roadmap.includes('Current working gate: V41 Gate 9'), + 'Roadmap must name V41 Gate 5 or later as current working gate.', ); assertCheck( failures, roadmap.includes('Next queued gate after V41 Gate 5: V41 ReadFitsFindingSynthesis Prompt Rewrite Search And AssetPack Context Hardening.') || roadmap.includes('Next queued gate after V41 Gate 6: V41 Conversation Tool And Interface Prompt Rewrite.') || - roadmap.includes('Next queued gate after V41 Gate 7: V41 Prompt Benchmark Report And Telemetry Integration.'), - 'Roadmap must preserve V41 Gate 6 or Gate 7 progression.', + roadmap.includes('Next queued gate after V41 Gate 7: V41 Prompt Benchmark Report And Telemetry Integration.') || + roadmap.includes('Next queued gate after V41 Gate 8: V41 Promotion Readiness.'), + 'Roadmap must preserve V41 Gate 6, Gate 7, Gate 8, or Gate 9 progression.', ); assertCheck(failures, roadmap.includes('V43+ agentic depositing'), 'Roadmap must preserve V43+ agentic depositing note.'); diff --git a/scripts/check-v41-gate6-readfitsfinding-prompt-hardening.mjs b/scripts/check-v41-gate6-readfitsfinding-prompt-hardening.mjs index 3275948a2..6a53f1980 100644 --- a/scripts/check-v41-gate6-readfitsfinding-prompt-hardening.mjs +++ b/scripts/check-v41-gate6-readfitsfinding-prompt-hardening.mjs @@ -248,14 +248,18 @@ function main() { const roadmap = fileExists(root, 'SPECIFICATIONS_ROADMAP.md') ? read(root, 'SPECIFICATIONS_ROADMAP.md') : ''; assertCheck( failures, - roadmap.includes('Current working gate: V41 Gate 6') || roadmap.includes('Current working gate: V41 Gate 7'), - 'Roadmap must name V41 Gate 6 or Gate 7 as current working gate.', + roadmap.includes('Current working gate: V41 Gate 6') || + roadmap.includes('Current working gate: V41 Gate 7') || + roadmap.includes('Current working gate: V41 Gate 8') || + roadmap.includes('Current working gate: V41 Gate 9'), + 'Roadmap must name V41 Gate 6 or later as current working gate.', ); assertCheck( failures, roadmap.includes('Next queued gate after V41 Gate 6: V41 Conversation Tool And Interface Prompt Rewrite.') || - roadmap.includes('Next queued gate after V41 Gate 7: V41 Prompt Benchmark Report And Telemetry Integration.'), - 'Roadmap must name V41 Gate 7 or Gate 8 as next.', + roadmap.includes('Next queued gate after V41 Gate 7: V41 Prompt Benchmark Report And Telemetry Integration.') || + roadmap.includes('Next queued gate after V41 Gate 8: V41 Promotion Readiness.'), + 'Roadmap must name V41 Gate 7, Gate 8, or Gate 9 as next.', ); assertCheck(failures, roadmap.includes('V41 Gate 6 closure anchor'), 'Roadmap must preserve V41 Gate 6 closure anchor.'); assertCheck(failures, roadmap.includes('V43+ agentic depositing'), 'Roadmap must preserve V43+ agentic depositing note.'); diff --git a/scripts/check-v41-gate7-conversation-tool-interface-prompt-rewrite.mjs b/scripts/check-v41-gate7-conversation-tool-interface-prompt-rewrite.mjs index e438ec053..95a319295 100644 --- a/scripts/check-v41-gate7-conversation-tool-interface-prompt-rewrite.mjs +++ b/scripts/check-v41-gate7-conversation-tool-interface-prompt-rewrite.mjs @@ -268,8 +268,19 @@ function main() { assertCheck(failures, packageJson.includes('check:v41-gate7'), 'package.json must expose check:v41-gate7.'); const roadmap = fileExists(root, 'SPECIFICATIONS_ROADMAP.md') ? read(root, 'SPECIFICATIONS_ROADMAP.md') : ''; - assertCheck(failures, roadmap.includes('Current working gate: V41 Gate 7'), 'Roadmap must name V41 Gate 7 as current working gate.'); - assertCheck(failures, roadmap.includes('Next queued gate after V41 Gate 7: V41 Prompt Benchmark Report And Telemetry Integration.'), 'Roadmap must name V41 Gate 8 as next.'); + assertCheck( + failures, + roadmap.includes('Current working gate: V41 Gate 7') || + roadmap.includes('Current working gate: V41 Gate 8') || + roadmap.includes('Current working gate: V41 Gate 9'), + 'Roadmap must name V41 Gate 7 or later as current working gate.', + ); + assertCheck( + failures, + roadmap.includes('Next queued gate after V41 Gate 7: V41 Prompt Benchmark Report And Telemetry Integration.') || + roadmap.includes('Next queued gate after V41 Gate 8: V41 Promotion Readiness.'), + 'Roadmap must name V41 Gate 8 or Gate 9 as next.', + ); assertCheck(failures, roadmap.includes('V41 Gate 7 closure anchor'), 'Roadmap must preserve V41 Gate 7 closure anchor.'); assertCheck(failures, roadmap.includes('V43+ agentic depositing'), 'Roadmap must preserve V43+ agentic depositing note.'); diff --git a/scripts/check-v41-gate8-prompt-program-benchmark-report.mjs b/scripts/check-v41-gate8-prompt-program-benchmark-report.mjs new file mode 100644 index 000000000..f6567606f --- /dev/null +++ b/scripts/check-v41-gate8-prompt-program-benchmark-report.mjs @@ -0,0 +1,299 @@ +#!/usr/bin/env node + +import { execFileSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const defaultRepoRoot = path.resolve(__dirname, '..'); +const ARTIFACT_PATH = '.bitcode/v41-prompt-program-benchmark-report.json'; + +const REQUIRED_ROW_IDS = [ + 'post-rewrite-prompt-inventory-delta', + 'readneed-post-rewrite-benchmark-delta', + 'readfitsfinding-post-rewrite-benchmark-delta', + 'conversation-interface-post-rewrite-benchmark-delta', + 'registry-lineage-version-telemetry-binding', + 'failsafe-thricified-inference-receipt-projection', + 'rich-stream-source-safe-telemetry-projection', + 'repair-hooks-parsed-output-redaction-posture', + 'gate8-tests-docs-workflows', +]; + +const REQUIRED_METRIC_IDS = [ + 'post_rewrite_promptpart_delta', + 'post_rewrite_prompt_delta', + 'benchmark_fixture_result_projection', + 'prompt_lineage_registry_versioning', + 'failsafe_thricified_receipt_projection', + 'parsed_output_schema_verdict_projection', + 'rich_stream_prompt_telemetry_projection', + 'repair_hook_redaction_posture', + 'source_safe_benchmark_report_disclosure', + 'depository_search_embedding_query_projection', +]; + +const REQUIRED_TELEMETRY_RECEIPT_IDS = [ + 'ReadNeed', + 'ReadFitsFindingSynthesis', + 'ConversationStreamEvent', + 'ReadingPipelineTelemetryProjection', + 'ReadingOperationalTelemetryEvent', + 'PromptPartBenchmarkResult', + 'PromptBenchmarkResult', + 'failsafe', + 'thricified_generation', + 'interpolated_prompt', + 'raw_response', + 'parsed_output', + 'schema_verdict', + 'repair', +]; + +const SECRET_MARKERS = [ + `${['sk', 'proj'].join('-')}-`, + `${['sb', 'secret'].join('_')}__`, + ['service', 'role'].join('_'), + ['eyJhbGciOi', 'JIUzI1Ni'].join(''), + ['OPENAI', 'API', 'KEY'].join('_'), + ['SUPABASE', 'SERVICE', 'ROLE'].join('_'), + ['VERCEL', 'TOKEN'].join('_'), + ['VERCEL', 'OIDC', 'TOKEN'].join('_'), + ['PRIVATE', 'KEY'].join('_'), +]; + +function parseArgs(argv) { + const args = { + skipBranchCheck: false, + skipPackageTests: false, + repoRoot: defaultRepoRoot, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--skip-branch-check') args.skipBranchCheck = true; + else if (arg === '--skip-package-tests') args.skipPackageTests = true; + else if (arg === '--repo-root') args.repoRoot = path.resolve(argv[++index]); + else if (arg === '--help' || arg === '-h') args.help = true; + else throw new Error(`Unknown argument ${arg}`); + } + + return args; +} + +function printHelp() { + process.stdout.write( + [ + 'Usage: node scripts/check-v41-gate8-prompt-program-benchmark-report.mjs [--skip-branch-check] [--skip-package-tests] [--repo-root ]', + '', + 'Checks V41 Gate 8 source-safe prompt-program benchmark report and telemetry integration.', + ].join('\n'), + ); + process.stdout.write('\n'); +} + +function read(root, relativePath) { + return readFileSync(path.join(root, relativePath), 'utf8'); +} + +function fileExists(root, relativePath) { + return existsSync(path.join(root, relativePath)); +} + +function git(root, args) { + return execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim(); +} + +function run(root, command, args) { + return execFileSync(command, args, { + cwd: root, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); +} + +function assertCheck(failures, condition, message) { + if (!condition) failures.push(message); +} + +function includesAll(values, requiredValues) { + return requiredValues.every((value) => values.includes(value)); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + printHelp(); + return; + } + + const root = args.repoRoot; + const failures = []; + const pointer = read(root, 'BITCODE_SPEC.txt').trim(); + + assertCheck( + failures, + pointer === 'V40', + `BITCODE_SPEC.txt must remain V40 during V41 gate work. Observed ${pointer || 'empty'}.`, + ); + + if (!args.skipBranchCheck) { + const branch = git(root, ['branch', '--show-current']); + assertCheck( + failures, + branch === 'version/v41' || /^v41\/gate-(?:8|9|10)-[a-z0-9][a-z0-9-]*$/u.test(branch), + `V41 Gate 8+ work must occur on version/v41 or v41/gate-8..10-* branches. Observed ${branch || 'detached HEAD'}.`, + ); + } + + const requiredFiles = [ + ARTIFACT_PATH, + '.bitcode/v38-prompt-benchmark-report.json', + '.bitcode/v38-disclosure-boundary-report.json', + '.bitcode/v39-operational-telemetry-repair-readback.json', + '.bitcode/v40-prompt-benchmark-smoke-v41-readiness.json', + '.bitcode/v41-promptpart-prompt-inventory.json', + '.bitcode/v41-registry-interpolation-contracts.json', + '.bitcode/v41-reading-prompt-benchmark-baselines.json', + '.bitcode/v41-readneed-prompt-hardening.json', + '.bitcode/v41-readfitsfinding-prompt-hardening.json', + '.bitcode/v41-conversation-tool-interface-prompt-rewrite.json', + 'packages/protocol/src/canonical/v41-prompt-program-benchmark-report.js', + 'packages/protocol/test/v41-prompt-program-benchmark-report.test.js', + 'scripts/generate-v41-prompt-program-benchmark-report.mjs', + 'scripts/check-v41-gate8-prompt-program-benchmark-report.mjs', + 'packages/prompts/src/benchmarking/runner.ts', + 'packages/prompts/src/benchmarking/types.ts', + 'packages/pipelines/asset-pack/src/reading-pipeline-observability.ts', + 'packages/pipelines/asset-pack/src/reading-operational-telemetry-repair-readback.ts', + 'packages/api/src/conversations/stream-events.ts', + 'packages/api/src/conversations/telemetry.ts', + 'uapi/components/base/bitcode/execution/pipeline-execution-log.tsx', + 'uapi/components/base/bitcode/execution/pipeline-execution-log-header.tsx', + 'BITCODE_SPEC_V41.md', + 'BITCODE_SPEC_V41_DELTA.md', + 'BITCODE_SPEC_V41_NOTES.md', + 'BITCODE_SPEC_V41_PARITY_MATRIX.md', + 'SPECIFICATIONS_ROADMAP.md', + 'README.md', + 'packages/protocol/README.md', + 'package.json', + '.github/workflows/bitcode-gate-quality.yml', + '.github/workflows/bitcode-canon-quality.yml', + ]; + + for (const relativePath of requiredFiles) { + assertCheck(failures, fileExists(root, relativePath), `Missing V41 Gate 8 file: ${relativePath}`); + } + + if (failures.length === 0) { + try { + run(root, 'node', ['scripts/generate-v41-prompt-program-benchmark-report.mjs', '--check']); + } catch (error) { + failures.push(`V41 prompt-program benchmark report artifact check failed: ${error.stderr || error.message}`); + } + } + + if (!args.skipPackageTests && failures.length === 0) { + try { + run(root, 'node', [ + '--test', + '--test-force-exit', + 'packages/protocol/test/v41-prompt-program-benchmark-report.test.js', + ]); + } catch (error) { + failures.push(`V41 prompt-program benchmark report protocol test failed: ${error.stderr || error.message}`); + } + } + + const serializedArtifact = fileExists(root, ARTIFACT_PATH) ? read(root, ARTIFACT_PATH) : ''; + for (const marker of SECRET_MARKERS) { + assertCheck(failures, !serializedArtifact.includes(marker), `V41 prompt-program benchmark report artifact must not contain secret marker ${marker}.`); + } + + const artifact = serializedArtifact ? JSON.parse(serializedArtifact) : null; + if (artifact) { + assertCheck(failures, artifact.artifactId === 'v41-prompt-program-benchmark-report', 'Prompt-program benchmark report artifactId must match.'); + assertCheck(failures, artifact.schemaId === 'bitcode.v41.promptProgramBenchmarkReport.v1', 'Prompt-program benchmark report schemaId must match.'); + assertCheck(failures, artifact.version === 'V41' && artifact.currentTarget === 'V40', 'Prompt-program benchmark report must bind V41 over active V40.'); + assertCheck(failures, artifact.passed === true, 'Prompt-program benchmark report artifact must pass.'); + assertCheck( + failures, + artifact.sourceSafetyVerdict === 'source-safe-prompt-program-benchmark-telemetry-metadata', + 'Prompt-program benchmark report artifact must be source-safe metadata.', + ); + assertCheck(failures, includesAll(artifact.metricIds, REQUIRED_METRIC_IDS), 'Prompt-program benchmark report must expose every required metric id.'); + assertCheck(failures, includesAll(artifact.rows.map((row) => row.reportRowId), REQUIRED_ROW_IDS), 'Prompt-program benchmark report must cover every row.'); + assertCheck(failures, includesAll(artifact.coverage.telemetryReceiptIds, REQUIRED_TELEMETRY_RECEIPT_IDS), 'Prompt-program benchmark report must cover required telemetry receipts.'); + assertCheck(failures, artifact.coverage.rowCount === REQUIRED_ROW_IDS.length, 'Prompt-program benchmark report row count must match required rows.'); + assertCheck(failures, artifact.coverage.promptProgramArtifactCount >= 10, 'Prompt-program benchmark report must bind prompt-program artifacts.'); + assertCheck(failures, artifact.coverage.telemetryReceiptCount >= 20, 'Prompt-program benchmark report must bind telemetry receipts.'); + assertCheck(failures, artifact.coverage.sourceRootPresentCount === artifact.coverage.sourceRootCount, 'All prompt-program benchmark report source roots must exist.'); + assertCheck(failures, artifact.coverage.requiredPredicateCount >= 45, 'Prompt-program benchmark report must require at least 45 predicates.'); + assertCheck(failures, artifact.coverage.passedPredicateCount === artifact.coverage.requiredPredicateCount, 'Prompt-program benchmark report predicates must all pass.'); + assertCheck(failures, artifact.coverage.failedPredicateIds.length === 0, 'Prompt-program benchmark report must have no failed predicates.'); + assertCheck(failures, artifact.coverage.failingRowIds.length === 0, 'Prompt-program benchmark report must have no failing rows.'); + assertCheck(failures, artifact.coverage.dependenciesPassing === true, 'Prompt-program benchmark report dependencies must pass.'); + assertCheck(failures, artifact.coverage.v38PromptBenchmarkRowCount >= 7, 'V38 prompt benchmark dependency must remain bound.'); + assertCheck(failures, artifact.coverage.v38InferenceTelemetryLevelCount >= 13, 'V38 inference telemetry dependency must remain bound.'); + assertCheck(failures, artifact.coverage.v41ReadingBenchmarkBaselineRowCount >= 10, 'Gate 4 Reading benchmark dependency must remain bound.'); + assertCheck(failures, artifact.coverage.v41ReadNeedHardeningRowCount >= 7, 'Gate 5 ReadNeed dependency must remain bound.'); + assertCheck(failures, artifact.coverage.v41ReadFitsFindingHardeningRowCount >= 8, 'Gate 6 ReadFitsFinding dependency must remain bound.'); + assertCheck(failures, artifact.coverage.v41ConversationToolInterfaceRowCount >= 9, 'Gate 7 Conversation/tool/interface dependency must remain bound.'); + assertCheck(failures, /^v38-prompt-benchmark-report:[a-f0-9]{24}$/u.test(artifact.dependencyRoots.v38PromptBenchmarkReportRoot), 'V38 prompt benchmark root must be present.'); + assertCheck(failures, /^v38-inference-telemetry-disclosure-report:[a-f0-9]{24}$/u.test(artifact.dependencyRoots.v38InferenceTelemetryDisclosureRoot), 'V38 inference telemetry root must be present.'); + assertCheck(failures, /^v38-ptrr-failsafe-thricified-stack:[a-f0-9]{24}$/u.test(artifact.dependencyRoots.v38PtrrFailsafeThricifiedStackRoot), 'V38 PTRR stack root must be present.'); + assertCheck(failures, /^v38-read-fits-finding-search:[a-f0-9]{24}$/u.test(artifact.dependencyRoots.v38ReadFitsFindingSearchEmbeddingsRoot), 'V38 ReadFitsFinding search embedding root must be present.'); + assertCheck(failures, /^v39-operational-telemetry-repair-readback:[a-f0-9]{24}$/u.test(artifact.dependencyRoots.v39OperationalTelemetryRepairReadbackRoot), 'V39 operational telemetry root must be present.'); + assertCheck(failures, /^v40-prompt-benchmark-smoke-v41-readiness:[a-f0-9]{24}$/u.test(artifact.dependencyRoots.v40PromptBenchmarkSmokeRoot), 'V40 prompt benchmark smoke root must be present.'); + assertCheck(failures, /^v41-promptpart-prompt-inventory:[a-f0-9]{24}$/u.test(artifact.dependencyRoots.gate2InventoryRoot), 'Gate 2 dependency root must be present.'); + assertCheck(failures, /^v41-registry-interpolation-contract:[a-f0-9]{24}$/u.test(artifact.dependencyRoots.gate3RegistryInterpolationRoot), 'Gate 3 dependency root must be present.'); + assertCheck(failures, /^v41-reading-prompt-benchmark-baselines:[a-f0-9]{24}$/u.test(artifact.dependencyRoots.gate4ReadingPromptBenchmarkBaselineRoot), 'Gate 4 dependency root must be present.'); + assertCheck(failures, /^v41-readneed-prompt-hardening:[a-f0-9]{24}$/u.test(artifact.dependencyRoots.gate5ReadNeedPromptHardeningRoot), 'Gate 5 dependency root must be present.'); + assertCheck(failures, /^v41-readfitsfinding-prompt-hardening:[a-f0-9]{24}$/u.test(artifact.dependencyRoots.gate6ReadFitsFindingPromptHardeningRoot), 'Gate 6 dependency root must be present.'); + assertCheck(failures, /^v41-conversation-tool-interface-prompt-rewrite:[a-f0-9]{24}$/u.test(artifact.dependencyRoots.gate7ConversationToolInterfacePromptRewriteRoot), 'Gate 7 dependency root must be present.'); + assertCheck(failures, artifact.sourceSafety.sourceSafeMetadataOnly === true, 'Prompt-program benchmark report must be metadata-only.'); + assertCheck(failures, artifact.sourceSafety.rawPromptTextSerialized === false, 'Prompt-program benchmark report must not serialize raw prompt text.'); + assertCheck(failures, artifact.sourceSafety.rawInterpolatedPromptSerialized === false, 'Prompt-program benchmark report must not serialize interpolated prompts.'); + assertCheck(failures, artifact.sourceSafety.rawProviderResponseSerialized === false, 'Prompt-program benchmark report must not serialize provider responses.'); + assertCheck(failures, artifact.sourceSafety.protectedPromptSerialized === false, 'Prompt-program benchmark report must not serialize protected prompts.'); + assertCheck(failures, artifact.sourceSafety.protectedSourceVisible === false, 'Prompt-program benchmark report must not expose protected source.'); + assertCheck(failures, artifact.sourceSafety.unpaidAssetPackSourceVisible === false, 'Prompt-program benchmark report must not expose unpaid AssetPack source.'); + assertCheck(failures, artifact.sourceSafety.credentialsSerialized === false, 'Prompt-program benchmark report must not serialize credentials.'); + assertCheck(failures, artifact.sourceSafety.walletPrivateMaterialVisible === false, 'Prompt-program benchmark report must not expose wallet private material.'); + assertCheck(failures, artifact.sourceSafety.settlementPrivatePayloadVisible === false, 'Prompt-program benchmark report must not expose settlement private payloads.'); + assertCheck( + failures, + artifact.rows.every((row) => /^v41-prompt-program-benchmark-report-row:[a-f0-9]{24}$/u.test(row.rowRoot)), + 'Prompt-program benchmark report rows must have deterministic row roots.', + ); + assertCheck( + failures, + artifact.rows.every((row) => row.sourceSafeMetadataOnly === true && row.rawPromptTextSerialized === false && row.rawProviderResponseSerialized === false), + 'Prompt-program benchmark report rows must remain source-safe metadata only.', + ); + } + + const packageJson = fileExists(root, 'package.json') ? read(root, 'package.json') : ''; + assertCheck(failures, packageJson.includes('generate:v41-prompt-program-benchmark-report'), 'package.json must expose generate:v41-prompt-program-benchmark-report.'); + assertCheck(failures, packageJson.includes('check:v41-gate8'), 'package.json must expose check:v41-gate8.'); + + const roadmap = fileExists(root, 'SPECIFICATIONS_ROADMAP.md') ? read(root, 'SPECIFICATIONS_ROADMAP.md') : ''; + assertCheck(failures, roadmap.includes('Current working gate: V41 Gate 8'), 'Roadmap must name V41 Gate 8 as current working gate.'); + assertCheck(failures, roadmap.includes('Next queued gate after V41 Gate 8: V41 Promotion Readiness.'), 'Roadmap must name V41 Gate 9 as next.'); + assertCheck(failures, roadmap.includes('V41 Gate 8 closure anchor'), 'Roadmap must preserve V41 Gate 8 closure anchor.'); + assertCheck(failures, roadmap.includes('V43+ agentic depositing'), 'Roadmap must preserve V43+ agentic depositing note.'); + + if (failures.length > 0) { + process.stderr.write(`V41 Gate 8 check failed:\n${failures.map((failure) => `- ${failure}`).join('\n')}\n`); + process.exit(1); + } + + process.stdout.write( + `V41 Gate 8 check passed: rows=${artifact.coverage.rowCount} predicates=${artifact.coverage.passedPredicateCount} root=${artifact.artifactRoot}\n`, + ); +} + +main(); diff --git a/scripts/generate-v41-prompt-program-benchmark-report.mjs b/scripts/generate-v41-prompt-program-benchmark-report.mjs new file mode 100644 index 000000000..ed1bb880b --- /dev/null +++ b/scripts/generate-v41-prompt-program-benchmark-report.mjs @@ -0,0 +1,62 @@ +#!/usr/bin/env node + +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ARTIFACT_PATH, + buildV41PromptProgramBenchmarkReport, +} from '../packages/protocol/src/index.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const repoRoot = path.resolve(__dirname, '..'); +const GENERATED_AT = '2026-05-26T00:00:00.000Z'; + +function parseArgs(argv) { + return { + check: argv.includes('--check'), + write: argv.includes('--write') || !argv.includes('--check'), + }; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const artifact = buildV41PromptProgramBenchmarkReport({ + generatedAt: GENERATED_AT, + repoRoot, + }); + const outputPath = path.join(repoRoot, V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ARTIFACT_PATH); + const serialized = `${JSON.stringify(artifact, null, 2)}\n`; + + if (args.check) { + const current = readFileSync(outputPath, 'utf8'); + if (current !== serialized) { + process.stderr.write( + `${V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ARTIFACT_PATH} is stale. Run pnpm run generate:v41-prompt-program-benchmark-report.\n`, + ); + process.exitCode = 1; + return; + } + if (!artifact.passed) { + process.stderr.write( + `V41 prompt-program benchmark report failed:\n${artifact.failures.map((failure) => `- ${failure}`).join('\n')}\n`, + ); + process.exitCode = 1; + return; + } + process.stdout.write( + `V41 prompt-program benchmark report ok rows=${artifact.coverage.rowCount} predicates=${artifact.coverage.passedPredicateCount} root=${artifact.artifactRoot}\n`, + ); + return; + } + + mkdirSync(path.dirname(outputPath), { recursive: true }); + writeFileSync(outputPath, serialized); + process.stdout.write( + `Wrote ${V41_PROMPT_PROGRAM_BENCHMARK_REPORT_ARTIFACT_PATH} rows=${artifact.coverage.rowCount} predicates=${artifact.coverage.passedPredicateCount} root=${artifact.artifactRoot}\n`, + ); +} + +main();