From 7e413561b15cc82b7de744fb73557f1135b460ec Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 5 Aug 2026 22:44:54 -0400 Subject: [PATCH] Publish dashboard-data-20260805: Inkling debuts fourth, Qwen3.8-Max twenty-fifth of 29 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two additions in one board update. Inkling (Thinking Machines Lab's first model, open weights on day one under Apache 2.0) debuts at 83.8 within-$1 — fourth of 29 and the strongest US open-weight model, at $0.098 per household on a forced tool contract that parsed 1,984/1,984. Qwen3.8-Max (GA 2026-08-03, closed until its promised weights appear) debuts at 71.5 — twenty-fifth, below its predecessor Qwen3.7-max (73.6) — on the JSON contract with a 98,304-token completion cap; it reasons 36-40k tokens per whole-scenario request, the heaviest on the roster, and also parsed 1,984/1,984. The payload carries 7,125 decisive judge-annotation rows across 935 cases with zero reference suspects; the strict publish gate passes and all 27 incumbent modelStats are byte-identical to dashboard-data-20260724. The manifest's live artifact entry and audit-annotation sha advance in this commit, the hero snapshot label moves to 2026-08-05, and the leaderboard gains a Thinking Machines provider (icon masked from the company's own favicon so it follows currentColor in both themes). Also updates the gpt-5.6 cost-map test to assert litellm's map-internal price structure instead of equality with PRICE_OVERRIDES_PER_1M: OpenAI cut Terra/Luna prices on 2026-07-30, and the overrides deliberately keep the rates the recorded July runs were billed at so historical costUsd reconstructions stay byte-stable. Co-Authored-By: Claude Fable 5 --- .../us_audit_row_annotations.csv | 13150 ++++++++-------- app/src/components/Hero.tsx | 2 +- app/src/components/ProviderMark.tsx | 33 + app/src/data.artifact.json | 8 +- app/src/data.versions.json | 2 +- app/src/modelMeta.ts | 14 +- paper/snapshot/20260501/manifest.json | 10 +- policybench/config.py | 9 + policybench/model_cards.py | 39 + policybench/paper_results.py | 18 +- tests/test_eval_no_tools.py | 21 +- tests/test_model_cards.py | 2 + 12 files changed, 6948 insertions(+), 6360 deletions(-) diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv index 7e51fc4..cebe438 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv @@ -1,5139 +1,5517 @@ country,scenario_id,variable,model,failure_source,failure_subtype,reference_suspect,annotation -us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model treated the filer as a qualifying surviving spouse, used joint Social Security thresholds and deductions, and therefore included only $11,542 of Social Security. Single-filer treatment includes $18,542.70 and, with $18,341.33 itemized deductions plus the $6,000 senior deduction, yields $26,095.78 of taxable income." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $48,000 of tax-exempt veterans benefits, treated ESI premiums as an AGI deduction despite no wages, and then contradicted its own positive-tax calculations by reporting zero. The applicable computation excludes veterans benefits and taxes $26,095.78 after itemized and senior deductions." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model correctly selected single filing status but overstated taxable Social Security as the full 85% cap of $22,182 and used the standard deduction. The taxable Social Security amount is $18,542.70, and $18,341.33 of itemized deductions plus the $6,000 senior deduction leave $26,095.78 taxable." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model assigned qualifying-surviving-spouse joint treatment without the required dependent-child facts and then asserted that roughly $14,000 of taxable income produced no tax. The filer is single, and the correct income and deductions leave $26,095.78 subject to the 2026 single brackets." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's stated inputs imply tax near $3,300, but it submitted $1,176 without any corresponding bracket or credit computation. Using $18,542.70 of taxable Social Security, the allowed itemized and senior deductions, and the 2026 brackets produces $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model assumed qualifying-surviving-spouse status and applied joint thresholds, the joint standard deduction, and joint brackets. With no qualifying child listed, the filer is single; $18,542.70 of Social Security and the applicable deductions produce $26,095.78 of taxable income." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly used qualifying-surviving-spouse joint treatment and then submitted $3,204 despite its own calculation of about $2,023. Single-filer treatment and the traced itemized and senior deductions yield $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model included $22,182 of Social Security instead of $18,542.70 and used only a $17,350 aged standard deduction. It omitted the applicable $18,341.33 itemized deduction and separate $6,000 senior deduction, overstating taxable income." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed that the TCJA rules expired for 2026, introduced a personal exemption, and applied projected 10% and 15% brackets. It also treated the listed ESI premiums as deductible medical spending; the governing 2026 computation instead uses current brackets and $18,341.33 of itemized deductions plus the $6,000 senior deduction." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model stated taxable income of $37,990 but returned only $1,109, a result inconsistent with any applicable 2026 bracket calculation. The traced deductions reduce taxable income to $26,095.78, whose bracket tax is $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a nonexistent post-sunset personal exemption and pre-TCJA 10% and 15% brackets. The 2026 rules instead allow $18,341.33 of itemized deductions and the $6,000 senior deduction and tax the resulting $26,095.78 under the current brackets." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assumed reversion to pre-TCJA rules, subtracting a personal exemption and applying a 15% second bracket. It also limited itemized deductions to the stated property tax rather than the traced $18,341.33, so it did not reach the correct taxable income or tax." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model subtracted a personal exemption that does not apply and used the wrong 2026 tax structure. It also omitted the full $18,341.33 itemized deduction and separate $6,000 senior deduction used to derive $26,095.78 of taxable income." -us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model correctly computed single-filer taxable Social Security near $18,542 but assumed property taxes were capped at $10,000 and therefore selected the standard deduction. The applicable itemized deductions total $18,341.33, followed by the $6,000 senior deduction, reducing taxable income to $26,095.78." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that age-related and surviving-spouse deductions eliminated taxable income without calculating them. The filer has $50,437.11 of AGI and only $24,341.33 of applicable deductions, leaving $26,095.78 taxable rather than zero." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the aged standard deduction as large enough to offset pension, IRA, interest, and taxable Social Security income. The applicable itemized and senior deductions total $24,341.33 against $50,437.11 of AGI, so substantial taxable income remains." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model used surviving-spouse joint Social Security thresholds and a joint standard deduction, reducing taxable Social Security to $11,542 and taxable income to $3,586. The filer is single, with $18,542.70 of taxable Social Security and $26,095.78 of taxable income after the applicable deductions." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model approximated taxable Social Security and used an aged standard deduction plus the senior deduction rather than the traced $18,341.33 itemized deduction plus $6,000 senior deduction. Those exact inputs produce $26,095.78 of taxable income and $2,883.49 of tax." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model selected a $24,150 standard-and-senior deduction package instead of the applicable $24,341.33 total consisting of itemized deductions and the senior deduction. It also rounded the traced income inputs, causing its taxable income and bracket tax to be overstated." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model's $359 answer reflects surviving-spouse joint treatment and deductions that leave only a small 10% bracket amount. The filer is single, and $18,542.70 of taxable Social Security together with the applicable deductions leaves $26,095.78 taxable." -us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that senior deductions and credits exceed taxable income. The deductions total $24,341.33 against AGI of $50,437.11, and no nonrefundable credit eliminates the resulting $2,883.49 liability." -us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model mixed single income treatment with qualifying-surviving-spouse labeling, added a personal exemption, and applied obsolete 10% and 15% brackets. The correct 2026 computation uses $18,341.33 of itemized deductions, the $6,000 senior deduction, and current single brackets." -us,scenario_000,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset, treated ESI premiums as deductible medical expenses, and applied a 15% bracket. The traced itemized deduction is $18,341.33, the senior deduction is $6,000, and the current 2026 brackets tax the resulting $26,095.78 at $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response failed the required output contract." -us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the aged standard deduction plus senior deduction totaling $24,150 instead of the applicable $18,341.33 itemized deduction plus $6,000 senior deduction. That $191.33 deduction difference, together with exact traced income amounts, separates its result from the correct $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly used a $750 elderly credit despite acknowledging that income phases it out, invoked an inapplicable lifetime-learning credit, and then forced the liability to zero. The deductions do not wipe out taxable income, and no nonrefundable credit reduces the traced $2,883.49 tax." -us,scenario_000,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model excluded Social Security entirely even though provisional income of $50,520 exceeds the applicable single-filer thresholds, and it improperly used qualifying-surviving-spouse deductions. The taxable Social Security amount is $18,542.70, producing AGI of $50,437.11 before the itemized and senior deductions." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model treated the filer as a qualifying surviving spouse and used joint Social Security thresholds, deductions, and brackets. The household facts contain no dependent child supporting that filing status, so the single thresholds produce $18,542.70 of taxable Social Security rather than $11,542." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly included tax-exempt veterans benefits in gross income, treated ESI premiums as an above-the-line deduction despite no wages, and then discarded its own positive-tax calculations to report zero. The applicable calculation yields $50,437.11 of AGI and $26,095.78 of taxable income." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model correctly selected single filing status but taxed 85% of all Social Security instead of applying the statutory single-filer inclusion formula, overstating taxable Social Security by about $3,639. It also selected the standard deduction instead of the $18,341.33 itemized deduction before applying the separate $6,000 senior deduction." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model assigned qualifying-surviving-spouse treatment without a dependent child and used joint thresholds and deductions. It then asserted that roughly $14,000 of taxable income produced zero tax, omitting the tax generated by the applicable single-filer computation." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's stated computation produces about $3,300, but it submitted $1,176 without applying any identified deduction, bracket, or credit that bridges the difference. The actual itemized-plus-senior deductions leave $26,095.78 taxable and generate $2,883.49." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model assumed qualifying-surviving-spouse status without the required dependent child and therefore used joint Social Security thresholds, the joint standard deduction, and joint brackets. It also omitted the separate $6,000 senior deduction from its final computation." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's own stated taxable income of $20,226 at a 10% rate yields about $2,023, not the submitted $3,204. It also used unsupported qualifying-surviving-spouse treatment instead of the single-filer Social Security and deduction calculation." +us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model treated 85% of the entire Social Security benefit as taxable and used only an aged standard deduction. The correct Social Security inclusion is $18,542.70, and $18,341.33 of itemized deductions plus the $6,000 senior deduction reduce taxable income to $26,095.78." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA provisions expired for 2026 and introduced a personal exemption and pre-TCJA 15% bracket. It also constructed $21,076 of itemized deductions rather than the traced $18,341.33, so both taxable income and the rate schedule were wrong." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The submitted $1,109 does not follow from the model's stated $37,990 of taxable income under any applicable 2026 bracket calculation. The correct deduction computation instead leaves $26,095.78 taxable and produces $2,883.49." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and the pre-TCJA 10%/15% brackets. The 2026 calculation uses $18,341.33 of itemized deductions plus the $6,000 senior deduction and the applicable 10%/12% single brackets." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly applied reverted pre-TCJA rules, including a personal exemption and a 15% second bracket. It also deducted only real-estate taxes rather than the traced $18,341.33 itemized amount plus the separate $6,000 senior deduction." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model subtracted an inapplicable personal exemption and omitted the $6,000 senior deduction while using the wrong 2026 rate structure. The correct deductions total $24,341.33 and leave $26,095.78 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model correctly computed the single-filer Social Security inclusion but capped the property-tax deduction at $10,000 and therefore chose the standard deduction. The applicable itemized deduction is $18,341.33, followed by the separate $6,000 senior deduction." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that age-related deductions eliminated taxable income without computing them. They reduce $50,437.11 of AGI to $26,095.78 of taxable income, not zero." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the aged-filer deductions as large enough to offset all pension, IRA, interest, and taxable Social Security income. After the $18,341.33 itemized deduction and $6,000 senior deduction, $26,095.78 remains taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model used joint qualifying-surviving-spouse Social Security thresholds and deductions, producing only $11,542 of taxable Social Security. Single-filer thresholds produce $18,542.70, and the applicable itemized-plus-senior deduction calculation yields $2,883.49." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model approximated taxable Social Security and used an aged standard deduction plus the senior deduction. The traced calculation uses exact taxable Social Security of $18,542.70 and itemized deductions of $18,341.33, leaving $26,095.78 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model correctly identified the Social Security inclusion but selected $18,150 of standard and age deductions instead of the $18,341.33 itemized deduction. That $191.33 deduction difference reduces taxable income to $26,095.78 and tax to $2,883.49." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model's $359 answer reflects joint qualifying-surviving-spouse thresholds and deductions. Without a qualifying dependent child, the single-filer computation includes $18,542.70 of Social Security and leaves $26,095.78 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that senior deductions and credits exceeded taxable income. No credit eliminates the liability, and $26,095.78 remains taxable after all applicable deductions." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model mixed single-filer Social Security taxation with qualifying-surviving-spouse labeling, then applied a restored personal exemption and pre-TCJA 15% bracket. The applicable calculation instead uses $18,341.33 of itemized deductions, the $6,000 senior deduction, and 2026 single brackets." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly applied a post-sunset 15% bracket and calculated itemized deductions as $21,076 by treating all listed ESI premiums above the medical floor as deductible. The traced itemized deduction is $18,341.33, followed by the $6,000 senior deduction and applicable 10%/12% brackets." +us,scenario_000,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model selected the aged standard deduction rather than the slightly larger $18,341.33 itemized deduction. With the additional $6,000 senior deduction, taxable income is $26,095.78 rather than about $26,286." +us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not satisfy the submission contract." +us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model selected $18,150 of standard and age deductions because it compared that amount only with the $16,470 real-estate-tax input. The full itemized deduction is $18,341.33; adding the $6,000 senior deduction leaves $26,095.78 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model reduced its rough tax by an elderly credit despite simultaneously describing that credit as phased out, and then forced the result to zero. The applicable deductions leave $26,095.78 taxable, and no nonrefundable credit reduces the resulting $2,883.49 liability." +us,scenario_000,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model declared Social Security entirely nontaxable even though provisional income includes taxable retirement income, tax-exempt interest, and half of Social Security and totals $50,520. That formula makes $18,542.70 taxable, while the filer uses itemized deductions plus the senior deduction rather than a joint standard deduction." +us,scenario_000,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that the aged single standard deduction eliminated all taxable income without accounting for $18,542.70 of taxable Social Security and the other taxable retirement income. The complete calculation leaves $26,095.78 taxable after itemized and senior deductions." us,scenario_000,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_000,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no earned wages were provided and that Social Security benefits are not employee-payroll-taxable, but then submitted $1,534 instead of the zero implied by its own rule. Its numeric answer is consistent with applying payroll tax to a nonwage retirement-income component rather than limiting the tax base to listed wages and covered earned compensation." -us,scenario_000,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable payroll_tax value or explanation. The correct derivation uses the listed income categories and finds no wage, salary, or covered earned income, leaving the employee payroll tax base at zero." -us,scenario_001,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied a projected post-TCJA joint standard deduction plus personal exemptions totaling only $28,350, leaving $3,052.25 of taxable income. PolicyEngine's 2026 joint standard deduction exceeds the household's $31,402.21 AGI, so the model created taxable income by using the wrong deduction regime." -us,scenario_001,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used a $29,600 deduction amount made from a post-TCJA-style standard deduction and personal exemptions, which left $1,802.25 of taxable income. The correct computation applies the 2026 PolicyEngine standard deduction for joint filers, which is larger than the $31,402.21 AGI and reduces taxable income to zero." -us,scenario_001,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security by treating $11,588 of benefits as taxable instead of the $2,338 included in PolicyEngine's AGI, inflating AGI to $40,652. It also used a much too low post-TCJA-style senior standard deduction of about $20,400, while PolicyEngine's 2026 joint standard deduction eliminates taxable income entirely." -us,scenario_001,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly converted age 65+ into ABD Medicaid eligibility and then relied on medically needy or Medicare Savings Program concepts that are not the eligibility category PolicyEngine assigned. It also misapplied SSI-related counting by treating Social Security as largely excluded and concluding the couple passed the aged pathway, while the engine trace shows the SSI pathway fails both income and asset tests and the category is NONE." -us,scenario_001,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated being 71 as making the head potentially Medicaid eligible without applying any Virginia pathway test. The correct derivation checks actual categories: MAGI categories fail at 2.05 FPL, the SSI-related aged pathway fails, and no other category is identified." -us,scenario_001,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,The model incorrectly treated Medicare eligibility plus moderate income as qualifying the head through a Medicare Savings Program Medicaid pathway. PolicyEngine's category computation assigns NONE because the head is above the relevant MAGI limits and does not satisfy the SSI-related income and asset tests. -us,scenario_001,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly computed the two employee FICA components as $1,801.968 and $421.428 but submitted $4,412.328 instead of their sum. It added an extra amount inconsistent with its own stated no-spouse-wages and no-Additional-Medicare-Tax reasoning, effectively doubling the employee-side payroll tax calculation." -us,scenario_001,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model identified the right tax base and included Social Security plus Medicare with no Additional Medicare Tax, but it did not apply the full 7.65% employee FICA rate to $29,064. The correct computation is $1,801.97 of Social Security tax plus $421.43 of Medicare tax, while $2,090 is an undercalculated shortcut rather than the required component sum." -us,scenario_001,payroll_tax,grok-4.3,llm_error,other,False,"The model used the right 7.65% employee FICA rate and the right $29,064 wage base, but it rounded the result to $2,220 instead of carrying the payroll tax calculation to dollars and cents. PolicyEngine computes the Social Security and Medicare components separately and rounds to $1,801.97 + $421.43 = $2,223.39." -us,scenario_001,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model bypassed PolicyEngine's gross-income eligibility screen and treated the elderly household as qualifying under a net-income calculation after deductions. It also converted a mortgage balance into a shelter deduction even though no mortgage payment, rent, property tax, or utility expense was listed, producing an invented positive monthly benefit instead of applying the gross-income cutoff that yields $0." -us,scenario_001,spouse_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented an aged pathway based on the spouse's lack of own earnings and ignored that Medicaid eligibility is not determined on separate spousal income alone. It also contradicted its own MAGI calculation: the household's MAGI exceeds the expansion threshold, and the spouse has no SSI-related categorical route in the engine." -us,scenario_001,spouse_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model misapplied Virginia ABD Medicaid by treating age 65+ plus modest assets as enough for eligibility. PolicyEngine did not place the spouse in an ABD, SSI, blind, disabled, dependent, or MAGI Medicaid category, and the household MAGI is 2.05 x FPL, above the MAGI-based limits." -us,scenario_001,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model used a generic shortcut that a 69-year-old is potentially Medicaid eligible without applying any Virginia pathway. The correct computation requires an actual eligibility category, and the spouse qualifies through none of the MAGI or non-MAGI pathways." -us,scenario_001,spouse_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model confused Medicare age with Medicaid eligibility and asserted Medicare Savings Program eligibility without checking the applicable category and income test. PolicyEngine assigns the spouse no Medicaid category, and the household's income level of 2.05 x FPL blocks MAGI-based Medicaid eligibility." -us,scenario_001,spouse_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model added an unlisted 40-quarter work-history confirmation requirement to the PolicyEngine Medicare eligibility variable. Under the reference rule, the spouse's age 69 alone satisfies Medicare eligibility, so treating missing work credits or Social Security receipt as disqualifying produced the wrong 0 answer." -us,scenario_001,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model allowed only $21,484 of total Virginia deductions and exemptions, omitting part of the deduction stack available to two spouses over age 65. After excluding Social Security and applying both age deductions, the joint standard deduction, and both personal exemptions, no Virginia taxable income remains." -us,scenario_001,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model’s unexplained $560 liability implies that it left positive Virginia taxable income after applying only a partial set of deductions. The two age deductions, joint standard deduction, personal exemptions, and Virginia Social Security subtraction eliminate taxable income." -us,scenario_001,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the head’s Virginia age deduction by Social Security benefits and therefore allowed only the spouse’s $12,000 age deduction. Social Security is subtracted from Virginia income; it does not offset the head’s age deduction, so both qualifying age deductions apply and eliminate taxable income with the other deductions and exemptions." -us,scenario_001,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model said it applied the Social Security exclusion, both age deductions, and both personal exemptions but produced a positive tax inconsistent with that deduction stack. Those adjustments together with the joint standard deduction reduce Virginia taxable income to zero, leaving no amount to apply the tax brackets to." -us,scenario_001,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $15,225 of Social Security as Virginia income, used conflicting estimated standard deductions, and submitted a number that does not match any of its own bracket calculations. Virginia excludes Social Security, and the resulting wage-based income is fully offset by both age deductions, the joint standard deduction, and personal exemptions." -us,scenario_002,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model applied single-filer Social Security thresholds, a single-filer standard deduction, and single-filer brackets even though the household is a joint return. Under the married-joint Social Security calculation, only $4,390 of Social Security is taxable, and the joint deduction with two senior additions eliminates all taxable income." -us,scenario_002,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly included the $24,000 disability benefit in AGI, overstated taxable Social Security at $23,663, and used an obsolete deduction scheme combining a $15,650 standard deduction with personal exemptions. The correct AGI is $23,130, which is fully absorbed by the 2026 joint standard deduction and two senior additions." -us,scenario_002,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model treated the $24,000 disability benefit as taxable and overstated taxable Social Security at $23,663. Excluding the disability benefit and applying the joint Social Security formula yields $23,130 of AGI, below the applicable joint deduction for two spouses over 65." -us,scenario_002,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated all $44,080 of Social Security as taxable instead of applying the married-joint provisional-income formula. Only $4,390 of Social Security enters AGI, so AGI is $23,130 and the senior-adjusted joint standard deduction leaves no taxable income." -us,scenario_002,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model asserted $22,203 of taxable income after deductions, which requires overstating taxable income sources or understating the joint deduction. The correct income inclusion is $18,340 of pension, $400 of interest, and $4,390 of taxable Social Security, while the $24,000 disability benefit is excluded; the resulting $23,130 AGI is entirely deducted." -us,scenario_002,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model expressly included disability income and taxable Social Security in a positive joint taxable-income calculation. The disability benefit is excluded, only $4,390 of Social Security is taxable, and the joint standard deduction plus both age additions reduces the resulting $23,130 AGI to zero taxable income." -us,scenario_002,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly included the $24,000 disability benefit in AGI and calculated $37,468 of taxable Social Security from a provisional-income base containing that excluded benefit. The correct joint calculation includes only $4,390 of Social Security, producing $23,130 of AGI that is fully offset by the deduction for a joint return with both spouses over 65." -us,scenario_002,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Although the model ultimately excluded the disability benefit, it treated all $44,080 of Social Security as taxable merely because combined income crossed the joint threshold. Crossing the threshold does not make every dollar taxable: the joint formula yields $4,390 of taxable Social Security, and the resulting $23,130 AGI is below the senior-adjusted joint standard deduction." -us,scenario_002,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no wages were provided and that Social Security retirement is not subject to payroll tax, then contradicted that rule by imposing $1,926 on unspecified wage-equivalent income. The prompt requires unlisted numeric inputs to be treated as zero, so there is no taxable payroll base for Social Security, Medicare, Additional Medicare Tax, or Washington employee payroll taxes." -us,scenario_002,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age 75 plus no own income as sufficient for an aged Medicaid pathway, but PolicyEngine assigned the spouse medicaid_category = NONE. It also ignored the household MAGI income level of 2.90 FPL for MAGI categories and substituted an individual-income shortcut that the reference trace does not use for eligibility." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The submitted $27,049 contradicts the model's own $22,379 calculation and has no supporting computation. Its calculation also omitted the deductible traditional IRA contribution and used estimated rather than exact 2026 parameters." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model taxed only $23,384 of Social Security instead of $50,583.25 and improperly deducted a $23,926 desired 401(k) contribution despite zero wages. It also treated the 64-year-old head as over 65 and invented unidentified nonrefundable credits." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model denied the $1,118 traditional IRA deduction and therefore used AGI around $183,291 instead of $182,172.27. Its submitted $19,524 also contradicts both tax totals developed in its reasoning." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model rejected the deductible $1,118 traditional IRA contribution, leaving AGI and taxable income too high. It then applied estimated bracket thresholds rather than the exact 2026 MFJ schedule." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied the 2026 MFJ ordinary brackets incorrectly, claiming about $23,886 of ordinary tax on roughly $147,331 when that income remains in the 22% bracket. It also omitted the traditional IRA deduction that lowers taxable income to $149,972.27." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a guessed $30,000 standard deduction instead of $32,200 and rejected the traditional IRA deduction. Those errors raised its ordinary taxable income before it applied estimated bracket thresholds." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Although it applied the traditional IRA deduction, it used approximate income totals and brackets and then rounded the calculated liability to $22,000. The exact taxable income is $149,972.27 and the exact bracket calculation plus $564 preferential tax is $22,154.70." -us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a pre-TCJA regime with personal exemptions, a $15,000 standard deduction, and 10%/15%/25% brackets. The applicable 2026 computation instead uses the $32,200 MFJ standard deduction, no personal exemptions, and the current 10%/12%/22% schedule at this income." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a $16,000 standard deduction, restored personal exemptions, and applied pre-TCJA 15% and 25% ordinary brackets. The applicable deduction is $32,200 with no personal exemptions." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies it taxed approximately the household's AGI rather than subtracting the $32,200 standard deduction and the full above-the-line deductions. The correct taxable income is $149,972.27, not approximately $183,865." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed expiration of the governing current-law rate and deduction structure, substituted a smaller standard deduction plus personal exemptions, and used the wrong brackets. The 2026 calculation uses a $32,200 MFJ standard deduction and current 10%/12%/22% brackets at this income." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a post-sunset regime with a $16,100 standard deduction, personal exemptions, and pre-TCJA ordinary rates. The applicable 2026 rules provide a $32,200 MFJ standard deduction, no personal exemptions, and lower current-law brackets." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used a $30,800 standard deduction instead of $32,200 and omitted the deductible traditional IRA contribution. This overstated taxable income by about $2,518 before applying the brackets." -us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model incorrectly replaced the actual $1,582 net capital loss with a $3,000 loss, used the wrong deduction and tax schedule, and invented a $500 sick-spouse credit. Its submitted $49,523.45 also contradicts its own stated final calculation of $28,867." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated age-based and itemized deductions as sufficient to erase the liability even though neither spouse is 65 and the listed medical expenses do not exceed 7.5% of AGI. After the $32,200 standard deduction, $149,972.27 remains taxable." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model reduced the large taxable retirement and Social Security income far beyond the deductions allowed by the facts. The standard deduction and above-the-line deductions leave $149,972.27 taxable, so the liability cannot fall to $2,320." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $151,091 instead of $149,972.27, reflecting omission of the deductible traditional IRA contribution and imprecise income amounts. That excess taxable income caused the $246.30 overstatement." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The submitted tax corresponds to taxable income around $151,090.50 rather than the traced $149,972.27. The model failed to incorporate the full $2,699.98 of above-the-line deductions before applying the $32,200 standard deduction." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $151,091 instead of $149,972.27. It omitted part of the $2,699.98 above-the-line deductions, including the traditional IRA deduction, before applying the tax brackets." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $151,090.50 taxable-income figure is $1,118.23 above the traced $149,972.27, matching omission of the traditional IRA deduction and exact adjustment amounts. This directly produced the overstated tax." -us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated desired retirement contributions and other deductions as offsets capable of eliminating tax despite zero wages and limited deductible expenses. The household retains $149,972.27 of taxable income after all allowed deductions." -us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-sunset system with a small standard deduction, personal exemptions, and 10%/15%/25% brackets. The applicable 2026 system uses the $32,200 MFJ standard deduction, no personal exemptions, and current-law brackets." -us,scenario_003,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model substituted inflation-adjusted pre-TCJA personal exemptions, standard deduction, and ordinary brackets. Those rules do not govern this 2026 calculation, which uses the $32,200 MFJ standard deduction and current-law rate schedule." -us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model used projected pre-TCJA rules, including a $16,950 standard deduction, two personal exemptions, and 15%/25% ordinary brackets. The applicable computation uses a $32,200 standard deduction, no personal exemptions, and the current 10%/12%/22% brackets at this income." -us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model improperly denied the traditional IRA deduction and instead deducted $500 of auto-loan interest from taxable income. The correct adjustments total $2,699.98 and produce taxable income of $149,972.27 without a separate $500 car-interest deduction." -us,scenario_003,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the absence of wages as preventing a tax calculation and returned zero despite explicitly listed taxable IRA distributions, pension income, interest, dividends, and Social Security. Those sources produce $149,972.27 of taxable income after deductions." -us,scenario_003,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model failed to subtract the $2,699.98 of above-the-line deductions correctly, used an age-based premise even though the head is 64, and miscomputed ordinary tax. Its submitted $41,228.50 also contradicts its own stated total of $25,531.98." -us,scenario_003,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model replaced the actual Medicare age test with a near-term eligibility concept. Age 64 does not satisfy the age-65 Medicare threshold for the modeled year, and no non-age Medicare pathway is listed." -us,scenario_003,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model stated the correct rule that age 64 is below the Medicare eligibility age of 65, then submitted the opposite binary value. Its error is a parse/consistency failure within the substantive answer: the explanation supports not eligible, while the submitted value marks eligible." -us,scenario_003,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model projected Medicare eligibility from the future age-65 threshold instead of applying eligibility to the constant full-year age facts. The head remains age 64 in the 2026 facts, so the standard age pathway is not met and no alternative Medicare pathway is present." -us,scenario_003,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model invented W-2 wages from weekly hours and an assumed $10 hourly wage despite the prompt saying unlisted numeric inputs are 0. It then applied Social Security and Medicare tax to those fabricated wages, and its arithmetic also double-counted an unexplained $947.17 component after already computing FICA on the same estimated wage base." -us,scenario_003,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model annualized labor earnings from hours worked even though no hourly wage or wage amount was provided and unlisted numeric inputs are 0. Payroll tax applies only to actual taxable employee earnings, so applying 6.2% Social Security and 1.45% Medicare rates to inferred wages produced a nonzero liability where the wage base is $0." -us,scenario_003,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model correctly identified that Texas has no general state individual income tax, then overrode that rule by adding an unsupported $280 residual for a nonstandard state component. The requested variable excludes local income and payroll taxes, and Texas contributes $0 to state income tax before refundable credits for this household." -us,scenario_004,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model put the full $51,475 Social Security benefit into AGI as if it were ordinary taxable income, then applied the standard deduction to that inflated AGI. It also contradicted itself on the tax calculation, moving from about $2,787 to $3,280 to a submitted $5,885 without a valid credit or surtax step." -us,scenario_004,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly identified that taxable income is $0 after excluding Social Security under the provisional-income test and applying the standard deduction, then abandoned that computation. Its submitted $4,600 comes from inventing unlisted wage income from the 40-hour work facts, even though the prompt says unlisted numeric inputs are 0 and no wages are listed." -us,scenario_004,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $3,144 reflects taxing a positive amount of Social Security-related income after a standard deduction rather than applying the provisional-income exclusion. With provisional income below the married-filing-jointly threshold, none of the Social Security benefit is taxable and the remaining $1,065 of non-Social-Security income is below the standard deduction." -us,scenario_004,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model stated the correct Social Security rule, then computed taxable income as $21,590 by effectively subtracting the standard deduction from gross income that still included Social Security. Once Social Security is excluded from AGI, only $1,065 remains before the standard deduction, so taxable income and pre-refundable-credit federal income tax are both $0." -us,scenario_004,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model computed MAGI as $1,065 by effectively ignoring the head's $51,475 of Social Security retirement income and taxable interest, then compared that understated income to the 138% FPL adult expansion limit. The correct engine trace has MAGI at 2.39 times FPL, so New York's expansion Medicaid pathway is income-ineligible and no other Medicaid category applies." -us,scenario_004,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model overrode the age-65 Medicare rule with an unsupported appeal to unspecified “programs” and “internal logic.” The prompt lists age 62 and no Medicare-qualifying disability, ESRD, ALS, or other qualifying status, so the head fails the Medicare eligibility test." -us,scenario_004,head_medicare_eligible,gpt-5.4-nano,llm_error,other,False,"The model's reasoning states the correct Medicare result: age 62 is below the standard eligibility age and the expected modeled value is 0. It then submitted value = 1, making the final parsed answer contradict its own explanation rather than reflecting a substantive eligibility calculation." -us,scenario_004,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model treated Social Security retirement receipt at age 62 as conferring Medicare eligibility. Early Social Security retirement benefits do not make a person Medicare eligible before age 65 absent a qualifying disability, ESRD, ALS, or similar pathway, none of which is listed for this head." -us,scenario_004,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's explanation correctly states that unlisted wages are zero and that Social Security retirement income produces no employee payroll tax, which supports payroll_tax = 0. It then submitted 5108 anyway, so the failure is an internal output/explanation mismatch rather than a substantive tax-rule calculation." -us,scenario_004,spouse_medicaid_eligible,deepseek-v4-pro,llm_error,health_coverage,False,"The model invented a very low spouse MAGI of $1,065 and treated that as below 138% FPL for a two-person household. The engine's MAGI calculation is 2.39 times FPL, so the spouse fails the ACA expansion adult income test and has no other Medicaid category." -us,scenario_004,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated most of the head's Social Security benefits as taxable income for New York, even though New York excludes Social Security retirement benefits from state taxable income. It also applied a contradictory bracket calculation to only $4,121 of claimed taxable income, producing a tax far above what New York rates can generate on that base." -us,scenario_004,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly noted that New York excludes Social Security and that listed non-excluded income is near zero after deductions, then abandoned the prompt's rule that unlisted numeric inputs are zero by inventing wages from full-time hours. Its $3,200 answer comes from applying New York rates to an assumed wage-based income base rather than the household's listed income." -us,scenario_004,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,The model gave no usable computation and returned a positive New York income tax despite the listed New York taxable income being eliminated by the married-filing-joint standard deduction and exemptions. Its answer is consistent with applying NY rates to a fabricated positive taxable-income amount rather than excluding Social Security and treating unlisted wages as zero. -us,scenario_004,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model included the full Social Security retirement benefit in federal AGI for the New York taxable-income build-up and then produced a positive New York taxable income despite acknowledging Social Security is not taxed by New York. It also treated traditional IRA contributions as an add-back and applied brackets to an overstated taxable-income base, yielding a nonzero state tax where the New York base is fully eliminated." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model replaced the traced $5,163.84 above-the-line deduction and $33,269.77 taxable-income deduction with a $3,000 capital-loss deduction and an estimated $32,600 standard deduction. It then used estimated brackets and understated ordinary tax, although it correctly calculated the $11,688 preferential-rate component and omitted the required $5,721.98 NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented roughly $110,000 of mortgage interest from the mortgage balance even though no mortgage-interest expense was listed, treated employer premiums as nonrefundable credits, and never produced a coherent taxable-income computation. Those unsupported inputs led to the grossly overstated $249,914 liability." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model explicitly excluded NIIT even though PolicyEngine's requested output includes $5,721.98 of NIIT, then reported $91,458 despite its own regular-tax arithmetic totaling $102,907. Its submitted number therefore contradicts its stated computation as well as omitting a required tax component." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest using an assumed 7% rate and a $60,000 itemized deduction, contrary to the instruction that unlisted mortgage interest is zero. It also applied a 20% qualified-dividend rate instead of the traced 15% rate and produced a total unsupported by its component arithmetic." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model arithmetically overstated AGI as about $604,318; the income items it listed produce $538,418 under its own method. It also invented roughly $60,000 of itemized deductions from an unlisted mortgage-interest amount, so neither its taxable income nor its $141,000 tax follows from the household facts." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly treated the 2026 TCJA individual provisions as expired, applied pre-TCJA ordinary brackets, restored miscellaneous itemized deductions and personal-rule features, and constructed an AMT liability from those obsolete rules. It also invented mortgage interest using a 7% rate rather than using zero for the unlisted expense." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented a $28,000 mortgage-interest deduction and consequently used an unsupported $40,574 itemized deduction instead of the traced $33,269.77 taxable-income deduction. Its approximate bracket calculation then overstated the traced $89,095.92 ordinary tax." -us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model imposed a $131,247 AMT that the traced computation does not generate. Replacing the regular-tax calculation with that erroneous tentative minimum tax drove the answer above the required sum of $89,095.92 ordinary tax, $11,688 preferential tax, and $5,721.98 NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA expired for 2026 and applied uncapped SALT deductions, a $1 million mortgage-interest limit, and personal exemptions. Those obsolete provisions produced the wrong deduction base and understated ordinary tax by $1,079.92." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used AGI of $460,936 instead of following the trace through $541,928.34 of gross income, $5,163.84 of above-the-line deductions, and $503,494.75 of taxable income. Its number therefore rests on excluding or double-deducting taxable wage and investment-income components before applying the brackets." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model added an AMT component that is absent from the traced liability and began from an understated AGI of roughly $525,049. The requested amount is the sum of ordinary tax, preferential-rate tax, and NIIT, without the extra AMT it imposed." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model incorrectly treated the TCJA individual provisions as expired and applied uncapped SALT deductions and ordinary rates reaching 33%. Those obsolete 2026 assumptions changed both taxable income and the ordinary-tax schedule from the traced computation. -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model deducted employer-sponsored insurance premiums from wages a second time, producing $370,323 of taxable wages and AGI of $525,049. It also applied personal exemptions and a 20% qualified-dividend rate, while the trace uses $77,920 taxed at 15%." -us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,period_annualization,False,"The model used the 2025 $30,000 standard deduction and 2025 tax brackets for a 2026 calculation. It also omitted the $5,721.98 NIIT, so its $103,573.36 contains only its regular ordinary and qualified-dividend tax." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The $21,735 answer does not apply the progressive ordinary-income brackets to the traced $503,494.75 of taxable income. The correct components alone are $89,095.92 of ordinary tax, $11,688 of preferential-rate tax, and $5,721.98 of NIIT, none of which its submitted total captures." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked real-estate-tax, medical, and unspecified nonrefundable-credit reductions large enough to suppress tax to $75,657 without identifying any applicable credit. The trace instead allows $33,269.77 of taxable-income deductions and no credit reduction from the $106,505.90 component sum." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $32,200 standard deduction plus a separate $1,070 nonitemizer charitable deduction, yielding taxable income of $505,148 rather than the traced $503,494.75. That deduction mismatch and its estimated ordinary brackets overstated regular tax by $529.02, while its NIIT calculation was essentially aligned." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The answer implies an understatement of the ordinary-rate component after accounting for the standard deduction, qualified dividends, and NIIT it named. The trace produces $89,095.92 of ordinary tax before adding $11,688 of preferential tax and $5,721.98 of NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model stated that it used itemized deductions but did not identify them, and its total exceeds the traced component sum by $632.10. The correct deduction sequence yields $503,494.75 of taxable income and $89,095.92 of ordinary tax before the preferential tax and NIIT additions." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model's total exceeds the traced liability by $871.10 despite naming the correct broad components. Its answer therefore applies the 2026 ordinary brackets or deduction amount incorrectly; the ordinary component is $89,095.92, followed by exactly $11,688 of preferential tax and $5,721.98 of NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model inserted a qualified-business-income deduction even though the household has no listed qualified business income and invoked unspecified nonrefundable credits. It also relied on an inferred mortgage-interest deduction, producing a taxable-income base unrelated to the traced $503,494.75." -us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied a TCJA-sunset regime with Pease, personal exemptions, restored miscellaneous itemized deductions, and a large AMT. Those obsolete rules generated a tentative minimum tax of $125,199 instead of the traced $89,095.92 ordinary tax plus $11,688 preferential tax." -us,scenario_005,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model deducted $13,369 of employer-sponsored insurance premiums from wages a second time and restored a miscellaneous itemized deduction under an erroneous TCJA-sunset assumption. This reduced AGI to $525,049 and distorted taxable income before it applied the wrong ordinary-rate schedule." -us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so the answer failed the required output contract." -us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model substituted AGI of $538,418 and a $32,200 standard deduction for the trace's $541,928.34 gross income, $5,163.84 above-the-line deductions, and $33,269.77 taxable-income deductions. That shortcut produced $506,218 of taxable income and overstated ordinary tax by $871.44." -us,scenario_005,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented enough mortgage interest and other itemized deductions to erase taxable income, even though unlisted mortgage-interest expense is zero and the traced taxable-income deduction is only $33,269.77. Retirement contributions, the limited capital loss, and the listed deductions do not offset $541,928.34 of gross income." -us,scenario_005,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest using an assumed 5.75% rate, omitted the taxable state-and-local-tax refund from AGI, and netted the capital loss against qualified dividends to create only $67,009 of preferential-rate income. The trace instead taxes the full $77,920 of adjusted net capital gains at 15% and adds $5,721.98 of NIIT, which the model omitted." +us,scenario_000,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented an aged-individual Medicaid pathway based on unspecified income exclusions and treated age 77 plus allegedly low countable income as sufficient. The head qualifies through no Medicaid category in Texas, so satisfying the immigration rule and being elderly do not produce eligibility." +us,scenario_000,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that the household has no earned wages and therefore owes $0 of employee payroll tax, but submitted 1534 instead. That numeric value contradicts its own payroll-tax-base calculation and its required final-value statement." +us,scenario_000,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required structured-output contract." +us,scenario_001,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed the Tax Cuts and Jobs Act standard-deduction rules sunset for 2026 and substituted a $15,150 joint standard deduction plus personal exemptions. Under the applicable 2026 rules, the joint standard deduction alone exceeds the $31,402.21 AGI, leaving zero taxable income and zero tax." +us,scenario_001,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an obsolete $19,600 age-adjusted standard deduction and personal exemptions, producing $1,802.25 of taxable income. The applicable 2026 joint standard deduction exceeds the household’s entire $31,402.21 AGI, so no taxable income reaches the 10% bracket." +us,scenario_001,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model both overstated taxable Social Security as $11,588 instead of the trace-derived $2,338.21 and used an obsolete $20,400 post-sunset standard deduction. Correct AGI is $31,402.21, which is fully absorbed by the applicable 2026 joint standard deduction, yielding zero taxable income and zero tax." +us,scenario_001,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model incorrectly treated most Social Security retirement income as excluded under SSI rules and invoked medically needy and Medicare Savings Program pathways without establishing their requirements. The head fails the SSI-related income and asset tests and has no Medicaid eligibility category. +us,scenario_001,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model converted being age 71 and potentially eligible for an aged pathway into actual Medicaid eligibility without applying any pathway's income and asset tests. The engine finds no qualifying category, with MAGI at 2.05 times FPL and the SSI pathway unavailable." +us,scenario_001,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,The model conflated Medicare Savings Program assistance with Medicaid eligibility and declared qualification solely from age and its characterization of income as low. The head qualifies through no Medicaid pathway: the MAGI categories and SSI-related aged pathway both fail. +us,scenario_001,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly computed Social Security tax of $1,801.968 and Medicare tax of $421.428 but failed to add its own components: they total $2,223.396, not $4,412.328. Its submitted value directly contradicts its stated computation." +us,scenario_001,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model included both FICA components but applied an incorrect combined amount. The standard 6.2% Social Security and 1.45% Medicare rates on $29,064 produce $2,223.39, whereas $2,090 reflects an effective rate of only about 7.19%." +us,scenario_001,payroll_tax,grok-4.3,llm_error,other,False,"The model identified the correct 7.65% FICA rate but rounded the liability to $2,220 instead of calculating the requested annual amount. Multiplying $29,064 by 7.65% and applying component-level cent rounding yields $2,223.39." +us,scenario_001,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model incorrectly treated reaching full retirement age as an exemption from employee Social Security and Medicare taxes. Wages remain subject to FICA at ages 71 and 69, so the head’s $29,064 of wages generates $1,801.97 of Social Security tax and $421.43 of Medicare tax." +us,scenario_001,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model bypassed the two-person SNAP gross-income limit and proceeded directly to an estimated net-income calculation. It also treated the mortgage balance and auto-loan interest as shelter deductions without listed mortgage-interest, property-tax, utility, or rent expenses, then guessed a $300 monthly benefit instead of applying the eligibility threshold and benefit formula." +us,scenario_001,spouse_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented an aged-pathway rule under which the spouse's lack of individual income overrides the household income test. PolicyEngine assigns no aged or other non-MAGI category, while the spouse's 2.05-times-FPL MAGI also fails the expansion pathway." +us,scenario_001,spouse_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model declared ABD eligibility from age and modest assets without applying Virginia's actual non-MAGI eligibility requirements and income test. The spouse receives no SSI and qualifies for no aged, blind, disabled, or other non-MAGI category; low resources alone do not establish ABD Medicaid eligibility." +us,scenario_001,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model converted being age 69 and merely potentially eligible into actual Medicaid eligibility without testing any pathway. The spouse's MAGI is 2.05 times FPL and no non-MAGI category applies, yielding no Medicaid eligibility category." +us,scenario_001,spouse_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model invented Medicare Savings Program receipt from age and its unsupported characterization of household income as low, then treated that invented status as Medicaid eligibility. No Medicare Savings Program or other Medicaid pathway is established, and the spouse's 2.05-times-FPL MAGI exceeds the expansion threshold." +us,scenario_001,spouse_medicare_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly imposed a 40-quarter Medicare-covered work-history requirement on the spouse's age-based eligibility. Under the benchmark's PolicyEngine rule, the spouse's age of 69 satisfies the Medicare eligibility test by itself, yielding value = 1." +us,scenario_001,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model allowed only $21,484 of combined deductions and exemptions instead of applying the full deductions available to this married couple, including an age deduction for each spouse. The full deductions eliminate taxable income, so applying Virginia’s brackets to $9,918.25 was erroneous." +us,scenario_001,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model’s unexplained approximation left positive Virginia taxable income after deductions. Applying the joint standard deduction, two personal exemptions, and both spouses’ age deductions after excluding Social Security eliminates taxable income." +us,scenario_001,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly offset the head’s Virginia age deduction by Social Security benefits and therefore allowed an age deduction for only the spouse. Virginia’s Social Security subtraction and age-deduction calculation leave both age deductions available here, eliminating taxable income rather than leaving $6,704." +us,scenario_001,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model claimed to apply two age deductions but its $250.20 result necessarily retained positive taxable income, so it did not actually subtract the full joint standard deduction, personal exemptions, and two age deductions. Those deductions exceed Virginia adjusted gross income and produce no tax." +us,scenario_001,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model substituted $1,600 of age-related additions for Virginia’s income-tax age deductions available to two taxpayers age 65 or older. Applying both age deductions, along with the joint standard deduction and personal exemptions, eliminates taxable income." +us,scenario_001,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated all $15,225 of Social Security as Virginia income despite acknowledging that only a federal taxable portion enters federal AGI and that Virginia subtracts taxable Social Security. It also produced $739.15 from mutually inconsistent taxable-income and bracket calculations; excluding Social Security and applying the full joint, personal, and age deductions yields zero taxable income." +us,scenario_002,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model calculated the married couple as though the head were a single filer, using the $25,000/$34,000 single Social Security thresholds and only one person's standard deduction. Joint treatment yields only $4,390 of taxable Social Security and deductions exceeding the resulting $23,130 AGI." +us,scenario_002,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $24,000 disability benefit and $23,663 of Social Security in AGI, then substituted personal exemptions and an understated deduction for the applicable 2026 joint and senior deductions. The correct inclusions produce $23,130 of AGI, which is fully absorbed by deductions." +us,scenario_002,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model treated the $24,000 disability benefit as taxable and overstated taxable Social Security at $23,663 instead of $4,390 under the married-joint provisional-income formula. It also understated the deductions available to two spouses over age 65, leaving taxable income where none remains." +us,scenario_002,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated all $44,080 of Social Security as taxable rather than applying the married-joint provisional-income formula, which includes only $4,390. That error inflated AGI from $23,130 to $62,820 and created a positive taxable-income balance." +us,scenario_002,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model explicitly included taxable disability and an overstated taxable portion of Social Security, producing $22,203 of taxable income. Disability is excluded, only $4,390 of Social Security enters AGI, and the applicable joint and senior deductions reduce taxable income to zero." +us,scenario_002,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's stated calculation includes disability income and taxable Social Security without applying the exclusions and married-joint Social Security formula that produce $23,130 of AGI. The available joint, age-based, and senior deductions exceed that AGI, leaving no taxable income." +us,scenario_002,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly included the $24,000 disability benefit and calculated $37,468 of taxable Social Security; married-joint provisional income instead produces only $4,390 of taxable Social Security. It also omitted applicable senior deductions beyond its projected standard deduction, so it taxed $46,098 when taxable income is zero." +us,scenario_002,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model included the $24,000 disability benefit and $23,663 of Social Security in AGI instead of excluding disability and including only $4,390 of Social Security. It also omitted applicable senior deductions, whereas total deductions eliminate the correct $23,130 AGI." +us,scenario_002,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Although the model ultimately excluded disability, it treated all $44,080 of Social Security as taxable merely because provisional income exceeded the first married-joint threshold. The graduated Social Security formula includes only $4,390, yielding $23,130 of AGI, and the applicable joint and senior deductions eliminate taxable income." +us,scenario_002,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that no wages were provided and that retirement Social Security is outside the payroll-tax base, then contradicted those findings by inventing “taxable wage-equivalent income” and applying employee payroll taxes to it. Unlisted wages are zero, so every employee payroll-tax component is zero rather than $1,926." +us,scenario_002,spouse_medicaid_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model incorrectly tested the spouse using zero individual income instead of the income and household-unit treatment applicable to Medicaid eligibility. It then assumed that age 75 automatically supplied an aged pathway below its income limit, but PolicyEngine assigns the spouse no Medicaid category and returns ineligible." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's submitted $27,049 contradicts its own stated calculation of approximately $22,379. Its calculation also omitted the $1,118 traditional IRA deduction and used estimated rather than exact 2026 parameters." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model taxed only $23,384 of Social Security by applying 85% directly to benefits above $32,000, instead of using the two-tier Social Security formula, which yields $50,583.25 taxable. It also deducted an unsupported $23,926 traditional 401(k) contribution and incorrectly treated the 64-year-old head as age 65 or older." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model explicitly calculated tax near $22,652–$22,674 but submitted $19,524 without a computation supporting that reduction. It also rejected the $1,118 traditional IRA deduction included in the required AGI derivation." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model disallowed the $1,118 traditional IRA deduction, leaving AGI at $183,291 instead of $182,172.27. It then used approximate bracket thresholds and a $32,300 standard deduction rather than the exact $32,200 deduction and 2026 bracket parameters." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an incorrect ordinary-bracket computation, obtaining about $23,886 on roughly $147,331 of ordinary taxable income. The exact 2026 brackets produce $21,590.70 before adding the $564 preferential-rate tax, and the model also omitted the $1,118 IRA deduction." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model rejected the $1,118 IRA deduction and estimated the standard deduction as $30,000 instead of using $32,200. Those errors inflated both AGI and taxable income before it applied estimated bracket thresholds." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model approximated and rounded its tax to $22,000 instead of carrying the stated income calculation through the exact 2026 brackets. It also misstated pre-Social-Security income by $100, which propagated into its AGI." +us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied pre-TCJA personal exemptions and 10%/15%/25% brackets. The applicable 2026 computation uses the $32,200 joint standard deduction and the current 10%/12%/22% bracket structure, with no personal exemptions." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed a $16,000 standard deduction, $10,000 of personal exemptions, and pre-TCJA 10%/15%/25% brackets. The calculation instead uses a $32,200 joint standard deduction, no personal exemptions, and the applicable 2026 brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model described approximately $183,865 as taxable income even after deductions, exceeding the correct $182,172.27 AGI. Subtracting the $32,200 standard deduction yields $149,972.27 of taxable income before the ordinary and preferential-rate tax calculations." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA provisions had expired and substituted personal exemptions plus a smaller standard deduction. The applicable 2026 rules use the $32,200 joint standard deduction, no personal exemptions, and current-law brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used post-sunset pre-TCJA rules, including a $16,100 standard deduction, $10,100 of personal exemptions, and older rate brackets. The applicable computation uses a $32,200 standard deduction, no exemptions, and the 2026 current-law brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used a $30,800 standard deduction instead of $32,200 and omitted the $1,118 traditional IRA deduction. Those errors raised taxable income from $149,972.27 to $152,490.50." +us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model mis-netted the capital transactions by deducting the full $3,000 loss while also retaining the $1,080 short-term gain, and it invented a $500 sick-spouse credit. Its submitted $49,523.45 also contradicts its own stated final calculation of $28,867." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated age-related and itemized deductions as sufficient to erase the liability, although neither spouse is 65 and the listed medical costs do not exceed 7.5% of AGI. The $32,200 standard deduction leaves $149,972.27 taxable rather than zero." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model reduced the liability to $2,320 through unspecified retirement and itemized deductions. The medical expenses do not clear the 7.5%-of-AGI floor, and the correct above-the-line deductions plus standard deduction leave $149,972.27 taxable." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $151,091, reflecting omission of the $1,118 traditional IRA deduction and an imprecise standard-deduction calculation. The correct taxable income is $149,972.27 before applying the exact 2026 brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $22,400.71 calculation used an overstated taxable-income base by failing to carry through the $1,118 traditional IRA deduction. The correct AGI and standard deduction produce $149,972.27 of taxable income." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $151,091 rather than $149,972.27. That difference comes from omitting the $1,118 traditional IRA deduction before applying the $32,200 standard deduction." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used $151,090.50 of taxable income, exactly $1,118.23 above the traced taxable-income amount. It failed to apply the traditional IRA adjustment that, together with the capital-loss deduction, reduces AGI to $182,172.27." +us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated deductions and retirement contributions as eliminating tax on substantial taxable retirement and Social Security income. The allowed adjustments and $32,200 standard deduction still leave $149,972.27 of taxable income." +us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied pre-TCJA standard-deduction, personal-exemption, and 10%/15%/25% bracket rules. The applicable 2026 computation uses the $32,200 joint standard deduction, no personal exemptions, and current-law ordinary brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model substituted inflation-adjusted pre-TCJA personal exemptions, a $17,103 standard deduction, and pre-TCJA brackets. The applicable rules instead provide a $32,200 joint standard deduction, no exemptions, and the current 2026 brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model overstated AGI as about $184,371 by roughly $2,199 and consequently used $152,171 of taxable income. The traced adjustments reduce AGI to $182,172.27 and taxable income to $149,972.27." +us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model applied projected pre-TCJA rules, using a $16,950 standard deduction, personal exemptions, and 10%/15%/25% brackets. The applicable 2026 calculation uses a $32,200 standard deduction, no personal exemptions, and current-law brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model invented a $500 deduction for auto-loan interest and rejected the $1,118 traditional IRA deduction. Auto-loan interest is not part of this federal taxable-income derivation; the traced above-the-line deductions are the $1,582 capital loss and $1,118 IRA contribution." +us,scenario_003,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model defaulted to zero because wages were unlisted, ignoring the taxable IRA distributions, pension, Social Security, interest, and dividends. Those nonwage income sources produce $182,172.27 of AGI and $149,972.27 of taxable income." +us,scenario_003,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model failed to subtract the $1,582 capital loss and $1,118 IRA deduction from gross income, and it incorrectly treated the 64-year-old head as age 65 or older. Its submitted $41,228.50 also contradicts its own stated total tax of $25,531.98." +us,scenario_003,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model's unexplained $21,902.90 liability understates the tax generated by the exact 2026 brackets on $146,212.27 of ordinary taxable income plus $3,760 taxed preferentially. That computation yields $21,590.70 of ordinary tax and $564 of preferential-rate tax." +us,scenario_003,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model replaced current-year Medicare eligibility with an invented near-term eligibility rule. At age 64 throughout 2026 and with no disability or other qualifying status, the head does not satisfy Medicare eligibility." +us,scenario_003,head_medicare_eligible,gpt-5.4-mini,llm_error,other,False,"The model's reasoning correctly states that age 64 is below the Medicare eligibility age and that the head is not eligible, but it submitted value = 1. This is an internal answer-label mismatch: the stated conclusion requires value = 0." +us,scenario_003,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a projection from age 64 to future eligibility at 65 even though demographic facts are constant throughout the tax year. Medicare eligibility is evaluated for 2026, and the head remains below age 65 with no disability or other qualifying pathway." +us,scenario_003,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model violated the instruction to treat unlisted numeric inputs as zero by inventing a $10 hourly wage and deriving $21,840 of wages from weekly hours. With no listed wages or other payroll-taxable earnings, Social Security and Medicare employee taxes are both zero; its stated $2,618.93 total also adds an unexplained $947.17 after already computing the two applicable components." +us,scenario_003,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model converted weekly hours into labor earnings despite the absence of any wage or hourly-rate input. Weekly hours alone do not create a payroll tax base, and all listed income consists of retirement, pension, IRA, and investment income that is not subject to employee Social Security or Medicare tax." +us,scenario_003,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model correctly identified that Texas has no general individual income tax, then invented a $280 residual from an unspecified nonstandard component. No such component applies in the state-income-tax output, so the 51-jurisdiction aggregation consists entirely of zeros and yields $0." +us,scenario_004,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model first counted the entire $51,475 Social Security benefit in AGI and later asserted that approximately $36,571 was taxable without applying the married-filing-jointly provisional-income thresholds. It also produced mutually inconsistent tax calculations of $2,787, $3,280, and $5,885; excluding the nontaxable Social Security leaves only net non-Social-Security income that the standard deduction fully offsets." +us,scenario_004,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly derived zero taxable income from the listed facts, then discarded that result and invented unlisted wage income from the adults' reported work hours. The prompt expressly sets unlisted wages to zero, so its unsupported conservative approximation of $4,600 has no tax base." +us,scenario_004,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The $3,144 answer implies that the model included a portion of Social Security in taxable income after the standard deduction. The married-filing-jointly provisional-income calculation excludes the Social Security benefit here, and the remaining interest net of the partnership loss is below the standard deduction, producing zero taxable income." +us,scenario_004,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly found that none of the Social Security benefit was taxable, but then subtracted the standard deduction from $52,540, a total that still included the full benefit. Once Social Security is excluded, the relevant income is $1,200 of interest minus the $135 partnership loss, which the joint standard deduction entirely eliminates." +us,scenario_004,head_medicaid_eligible,deepseek-v4-pro,llm_error,health_coverage,False,"The model reduced MAGI to $1,065 by omitting the head’s Social Security retirement benefits from Medicaid MAGI. Medicaid MAGI includes tax-exempt Social Security, producing approximately $52,540 or 2.39 times FPL, above New York’s expansion-adult limit; the head also qualifies through no non-MAGI pathway." +us,scenario_004,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model acknowledged the standard age-65 threshold but invented an unspecified alternative status instead of applying the supplied facts. At age 62 with no disability or other Medicare-qualifying status, the head is not eligible." +us,scenario_004,head_medicare_eligible,gpt-5.4-nano,llm_error,other,False,"The model's reasoning explicitly concluded that the head was not Medicare eligible and that the expected output was 0, but it submitted value = 1. This is an internally contradictory final-output contract failure." +us,scenario_004,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model incorrectly treated receipt of Social Security retirement benefits at age 62 as conferring Medicare eligibility. Retirement-benefit receipt does not replace the age-65 requirement, and no disability or other qualifying pathway was listed." +us,scenario_004,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model applied the age threshold backward by asserting that age 62 meets a requirement of age 65 or older. Because 62 is below 65 and no alternative qualifying status was supplied, the correct binary result is 0." +us,scenario_004,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model’s reasoning correctly identified a zero payroll-tax base but then submitted $5,108, contradicting its own derivation. That amount applies payroll tax to nonexistent covered earnings even though the stated Social Security retirement income is not subject to employee Social Security or Medicare tax and unlisted wages are zero." +us,scenario_004,spouse_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model calculated $1,065 by netting $1,200 of interest against the $135 partnership loss and omitting the head’s $51,475 Social Security retirement income from Medicaid MAGI. Medicaid MAGI adds Social Security benefits, yielding 2.39 times FPL; the 51-year-old spouse therefore fails the expansion-adult income limit and has no categorical pathway." +us,scenario_004,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly carried a federally taxable portion of Social Security into New York income instead of applying New York's subtraction for Social Security benefits. It also miscomputed tax on its own stated $4,121 taxable income: New York's lowest bracket would produce a small fraction of $1,380." +us,scenario_004,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly identified that New York excludes Social Security and that the listed non-excluded income is eliminated by the joint standard deduction, but then invented roughly $90,000 of wages from the spouses' work hours. The prompt expressly sets unlisted wage amounts to zero, so those assumed wages cannot enter New York taxable income." +us,scenario_004,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $1,582 implies a positive New York taxable-income base that does not exist. After excluding Social Security, the $1,200 interest less the $135 partnership loss is fully absorbed by the joint standard deduction, yielding zero tax." +us,scenario_004,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $51,475 of Social Security as federal AGI and failed to remove it consistently when deriving New York taxable income, despite stating that New York does not tax it. It also incorrectly added back the $721 traditional IRA contribution; with Social Security excluded, the remaining $1,065 is eliminated by the joint standard deduction." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model deducted the full $46,308 of desired 401(k) contributions instead of the $41,650 allowed contribution and used an estimated $32,600 standard deduction instead of the traced $5,163.84 above-the-line and $33,269.77 taxable-income deductions. It also omitted the $5,721.98 NIIT, leaving its total materially understated." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented roughly $110,000 of mortgage interest from the mortgage balance even though unlisted expenses are zero, and its stated taxable-income arithmetic does not reconcile with its $249,914 answer. The traced tax components total only $89,095.92 ordinary tax, $11,688 preferential-rate tax, and $5,721.98 NIIT." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model deducted $46,308 of desired 401(k) contributions rather than the $41,650 allowed amount, included the $4,148 state refund, and substituted estimated deduction and bracket values for the 2026 parameters. It then explicitly excluded NIIT and submitted a figure even below its own stated $102,907 regular-tax calculation." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model fabricated mortgage interest using an assumed 7% rate despite the instruction that unlisted amounts are zero, then used an unsupported $60,000 itemized deduction. Its $154,430 answer also does not follow from its stated ordinary tax, $15,584 dividend tax, and separate NIIT components." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model made an arithmetic error by calling the listed components approximately $604,318 of AGI; its own income items produce $538,418 under its assumptions. It also invented mortgage interest and a roughly $60,000 itemized deduction instead of applying the traced deductions and exact 2026 tax computation." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly treated the TCJA individual provisions as expiring for 2026 and applied reconstructed pre-TCJA brackets, personal AMT rules, a restored miscellaneous deduction, and a $10,000 SALT cap. It also invented mortgage interest from an assumed rate; the applicable computation instead yields $503,494.75 of taxable income and $89,095.92 of ordinary tax before adding preferential-rate tax and NIIT." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented approximately $28,000 of mortgage interest and used a $40,574 itemized deduction, although no mortgage-interest expense was listed. It also deducted the full desired 401(k) contributions rather than the $41,650 allowed amount and did not separately calculate the $5,721.98 NIIT." +us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model imposed a $131,247 AMT rather than using the traced ordinary-tax calculation of $89,095.92. Its asserted $465,449 taxable income and AMT pathway replace the applicable $503,494.75 taxable income, preferential-rate tax, and NIIT computation." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration and deducted uncapped SALT, mortgage interest, and personal exemptions. Those deductions do not produce the traced $503,494.75 taxable income; applying the governing 2026 deductions and brackets raises ordinary tax from its $88,016 estimate to $89,095.92." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $460,936 AGI and referred to pension income even though the $70,000 pension is tax-exempt. The trace begins with $541,928.34 of gross income and reaches $503,494.75 of taxable income after the specified deductions." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used roughly $525,049 of AGI and added an AMT component, neither of which follows the traced computation. It effectively subtracted employer insurance premiums from wages again and replaced the applicable $89,095.92 ordinary-tax calculation with an unsupported AMT-inclusive estimate." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied expired-law assumptions—uncapped SALT and ordinary rates reaching 33%—instead of the governing 2026 parameters. This produced the wrong deduction base and ordinary tax rather than $503,494.75 of taxable income and $89,095.92 of bracket tax." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model double-subtracted $13,369 of employer-sponsored insurance premiums from wages, producing $525,049 instead of following the trace's wage and gross-income treatment. It also applied personal exemptions and a 20% qualified-dividend rate, while the traced preferential tax is $11,688, exactly 15% of $77,920." +us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used 2025 brackets and a $30,000 standard deduction instead of the applicable 2026 parameters and traced $33,269.77 taxable-income deduction. It also omitted the $5,721.98 NIIT entirely." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The $21,735 answer captures only a small fraction of the liability and is consistent with taxing preferential income while omitting most ordinary bracket tax and NIIT. The correct component calculation includes $89,095.92 of ordinary tax, $11,688 of preferential-rate tax, and $5,721.98 of NIIT." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model's $75,657 answer implies excessive deductions or omitted high-bracket ordinary income, while offering no reconciled income or tax components. The traced $503,494.75 taxable income alone generates $89,095.92 of ordinary tax before preferential-rate tax and NIIT." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model deducted the full $46,308 desired 401(k) amount, included the state refund, and applied a $32,200 standard deduction plus a separate $1,070 charitable deduction. The engine instead caps 401(k) contributions at $41,650 and applies $5,163.84 above the line plus $33,269.77 against taxable income, yielding $503,494.75." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The answer understates the liability despite identifying the standard tax components, so its approximate 2026 deduction or bracket calculation is wrong. The exact components are $89,095.92 ordinary tax, $11,688 preferential-rate tax, and $5,721.98 NIIT." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model invoked itemized deductions without identifying or reconciling them to the traced $33,269.77 taxable-income deduction. Its estimate consequently overstates the exact $89,095.92 ordinary-tax component after the correct gross-income and deduction calculation." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the relevant components but used approximate deduction or bracket parameters that produced $107,377 instead of their exact sum. At $503,494.75 of taxable income, the components are $89,095.92, $11,688, and $5,721.98." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model applied a qualified-business-income deduction even though the household has no listed qualified business or self-employment income. It also invoked unspecified mortgage interest and credits, replacing the traced deduction and tax calculation with unsupported adjustments." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied post-sunset Pease, personal-exemption, miscellaneous-deduction, and AMT rules. Those assumptions generated a spurious $125,199 tentative AMT instead of the traced $89,095.92 ordinary tax plus $11,688 preferential-rate tax." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-subtracted $13,369 of employer health premiums from gross wages and deducted the full desired 401(k) contributions, producing the wrong $525,049 AGI. It also applied restored miscellaneous-itemized-deduction rules and calculated $106,958 of ordinary tax rather than the traced $89,095.92." +us,scenario_005,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model double-subtracted employer health premiums, deducted roughly $64,011 of SALT, and applied Pease and restored miscellaneous deductions. Those unsupported deductions and an added $18,221 AMT replace the traced path to $503,494.75 of taxable income and $89,095.92 of ordinary tax." +us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model deducted the full $46,308 desired 401(k) contribution, included the $4,148 state refund, and used a $32,200 standard deduction, producing $506,218 rather than $503,494.75 of taxable income. Those input and deduction errors raised ordinary tax above the traced $89,095.92." +us,scenario_005,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented enough mortgage interest and itemized deductions to erase taxable income, despite the instruction that unlisted expenses are zero. The listed deductions leave $503,494.75 taxable, which generates substantial ordinary tax, preferential-rate tax, and NIIT." +us,scenario_005,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest using an assumed 5.75% rate and mishandled capital-gain netting by treating $67,009 as preferential income; the net capital loss eliminates preferential LTCG, leaving $77,920 of qualified dividends. It also omitted the state refund and the $3,000 capital-loss deduction from its initial AGI arithmetic and failed to include NIIT." +us,scenario_005,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model used a $15,700 standard deduction, which is a single-filer-scale amount, even though the household files married jointly. Its stated gross income and $466,534 taxable income also fail to reconcile with the traced $541,928.34 gross income and $503,494.75 taxable income." us,scenario_005,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_005,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model used a projected $183,600 Social Security wage base instead of the applicable base and a 1.2% California SDI rate instead of the rate producing $5,590. Its submitted $33,296.55 also contradicts both of the totals stated in its own reasoning." -us,scenario_005,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model used an obsolete capped California SDI base, added employee ETT, and then improperly added $10,086.52 of state income tax to the payroll-tax output. It also used the wrong Social Security wage base; payroll tax contains $22,599 of Social Security and $5,590 of uncapped California SDI, not state income tax." -us,scenario_005,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model used an estimated $181,800 Social Security wage base and a 1.2% California SDI rate. The applicable calculations yield $22,599 of Social Security and $5,590 of SDI, producing $36,044 after Medicare taxes." -us,scenario_005,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model correctly derived $22,599 of Social Security, $6,235 of Medicare, and $1,620 of Additional Medicare Tax, but submitted a value below even that $30,454 federal subtotal and omitted the $5,590 California SDI contribution. Its numeric submission contradicts its own component arithmetic." -us,scenario_005,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model identified the correct $30,454 federal subtotal but failed to add the $5,590 California SDI liability despite claiming it was included. California SDI is a separate mandatory employee payroll-tax component, so the total is $36,044." -us,scenario_005,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model used $176,100 as the 2026 Social Security wage base and 1.2% for California SDI. Those assumptions understated both Social Security, which is $22,599, and uncapped SDI, which is $5,590." -us,scenario_005,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model used the wrong Social Security wage base and SDI rate, then submitted $27,306 even though its own successive calculations produced $29,691.40 and $34,421.40. The required components total $36,044, including $5,590 of California SDI." -us,scenario_005,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model understated Social Security as $22,407 rather than $22,599 and California SDI as $4,080 rather than $5,590. Regular and Additional Medicare were correct, but the two understated components reduced the total by $1,702." -us,scenario_005,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model used the wrong Social Security wage base, producing only $21,242, and applied a 1.1% California SDI rate, producing $4,730. The correct components are $22,599 of Social Security and $5,590 of SDI." -us,scenario_005,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The submitted amount does not include the full mandatory California SDI contribution alongside the federal payroll taxes. The federal components total $30,454 and uncapped California SDI adds $5,590, yielding $36,044." -us,scenario_005,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced wage bases by employer-sponsored insurance deductions, but the traced payroll-tax computation applies to the full $430,000 of gross wages. It consequently understated the Social Security, Medicare, Additional Medicare, and California SDI bases." -us,scenario_005,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model applied a 1.1% California SDI rate instead of the rate yielding $5,590 and did not use the applicable Social Security wage base yielding $22,599. These incorrect parameters understated the total payroll tax." -us,scenario_005,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model deducted employer-sponsored insurance premiums from gross wages for Social Security, Medicare, Additional Medicare Tax, and SDI. The traced computation uses the full $180,000 and $250,000 wage amounts, producing $30,454 federally plus $5,590 of California SDI." -us,scenario_005,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model imposed a $165,384 per-worker cap on California SDI and used an incorrect $182,100 Social Security wage base. California SDI is uncapped here and equals $5,590, while Social Security equals $22,599." -us,scenario_005,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model incorrectly asserted that both spouses were above the Social Security wage cap and omitted Additional Medicare Tax because it treated the regular Medicare rate as exhaustive. It also omitted California SDI; the correct computation includes $1,620 of Additional Medicare Tax and $5,590 of SDI." -us,scenario_005,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The submitted $13,905 is incompatible with the stated inclusion of Social Security, Medicare, Additional Medicare Tax, and California payroll taxes. Those components are $22,599, $6,235, $1,620, and $5,590 respectively, totaling $36,044." -us,scenario_005,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model used a projected $183,600 Social Security wage base, yielding $22,543.20 instead of $22,599, and a 1.2% SDI rate, yielding $5,160 instead of $5,590. These two parameter errors account for the understated total." -us,scenario_005,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model approximated California SDI at 1.3% and also used an incorrect estimated Social Security wage base. Exact application produces $22,599 of Social Security and $5,590 of SDI, with total payroll tax of $36,044 rather than $36,667." -us,scenario_005,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"The model approximated the 2026 Social Security wage base or California SDI contribution rather than applying the traced values. The exact components are $22,599 of Social Security, $6,235 of Medicare, $1,620 of Additional Medicare Tax, and $5,590 of SDI." -us,scenario_005,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model used an estimated California SDI amount or Social Security wage base, leaving the total $430 short. Exact computation adds $5,590 of SDI to the $30,454 federal subtotal for $36,044." -us,scenario_005,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The reasoning lists only federal OASDI, Medicare, and Additional Medicare Tax and omits California SDI as a distinct mandatory employee payroll tax. The federal subtotal is $30,454 and California SDI adds $5,590." -us,scenario_005,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model capped California SDI and calculated only $4,226, but SDI applies without that wage cap and equals $5,590. It also used an incorrect Social Security cap for the spouse rather than the base yielding $11,439." -us,scenario_005,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model subtracted $13,369 of employer-sponsored insurance premiums from wages before computing FICA and Additional Medicare Tax, although the traced bases are the full $430,000 of wages. It also imposed an obsolete $83,067 California SDI wage cap and used 1.1%, reducing SDI to $1,827 instead of $5,590." -us,scenario_005,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required output contract." -us,scenario_005,payroll_tax,kimi-k3,llm_error,payroll_tax_base,False,"The model correctly calculated the $30,454 federal subtotal but applied a 1.2% California SDI rate, yielding $5,160 rather than $5,590. The missing $430 of SDI explains the difference from $36,044." -us,scenario_005,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model incorrectly treated income-tax deductions and exemptions as offsets against employee payroll taxes. Social Security, Medicare, Additional Medicare Tax, and California SDI remain due on the wage bases and total $36,044; they are not reduced to zero by deductions." -us,scenario_005,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model substituted the 2025 Social Security wage base for 2026 and imposed an obsolete $153,164 California SDI cap at 1.1%. The applicable Social Security amount is $22,599 and uncapped California SDI is $5,590, not $3,369.60." +us,scenario_005,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model used a projected $183,600 Social Security wage base instead of $184,500 and a 1.2% California SDI rate instead of 1.3%. Its submitted $33,296.55 also contradicts its own corrected calculation, so it did not consistently aggregate the stated components." +us,scenario_005,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly capped California SDI wages and added California ETT and state income tax, neither of which belongs in this employee payroll-tax output. It also used the obsolete $168,600 Social Security wage base instead of the 2026 $184,500 base." +us,scenario_005,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $181,800 Social Security wage base and a 1.2% California SDI rate. The applicable inputs are a $184,500 Social Security wage base and uncapped California SDI at 1.3%, raising the total to $36,044." +us,scenario_005,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model correctly derived $22,599 of Social Security tax, $6,235 of Medicare tax, and $1,620 of Additional Medicare Tax, but then submitted less than even that federal subtotal and omitted California SDI. California SDI is 1.3% of the full $430,000, adding $5,590." +us,scenario_005,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"The model correctly identified the federal components but failed to add California SDI despite claiming it was included. Uncapped California SDI at 1.3% of $430,000 contributes $5,590, producing the $36,044 total." +us,scenario_005,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $176,100 as the 2026 Social Security wage base and 1.2% as the California SDI rate. The calculation requires the $184,500 Social Security base and the 1.3% uncapped SDI rate." +us,scenario_005,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model vacillated between excluding California SDI and including it at 1.1%, then submitted a number inconsistent with both calculations. California SDI is mandatory here, uncapped, and calculated at 1.3% of $430,000; Social Security also uses the $184,500 base." +us,scenario_005,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model understated both Social Security and California SDI, reporting $22,407 and $4,080 instead of $22,599 and $5,590. It therefore used incorrect 2026 wage-base or SDI-rate parameters." +us,scenario_005,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an undersized Social Security base, yielding $21,242 instead of $22,599, and applied a 1.1% California SDI rate instead of 1.3%. The correct SDI contribution is $5,590 on uncapped wages." +us,scenario_005,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The answer is consistent with aggregating federal FICA and Additional Medicare Tax while omitting or severely understating California SDI. The required state employee contribution is $5,590 on the full $430,000 wage base." +us,scenario_005,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced payroll-tax wage bases by employer-sponsored insurance deductions, but the traced calculation applies Social Security, Medicare, Additional Medicare Tax, and California SDI to the stated gross wages. That improper reduction understated multiple components." +us,scenario_005,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied California SDI at 1.1% rather than 1.3% and also understated the Social Security component. Correctly applying the $184,500 Social Security wage base and $5,590 of SDI yields $36,044." +us,scenario_005,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model subtracted employer-sponsored insurance premiums from both spouses' payroll-tax wages, which also reduced its Additional Medicare excess-wage calculation. The traced payroll taxes use the full $180,000 and $250,000 wages, with California SDI at 1.3% rather than 1.1%." +us,scenario_005,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"The model imposed a California SDI wage cap even though California SDI is uncapped for 2026, and it used a 1.2% rate instead of 1.3%. It also used a $182,100 Social Security wage base rather than $184,500." +us,scenario_005,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model incorrectly stated that both spouses were above the Social Security cap even though the head's $180,000 is below the $184,500 base. It also omitted the $1,620 Additional Medicare Tax and the $5,590 California SDI contribution." +us,scenario_005,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The submitted $13,905 cannot result from aggregating the named Social Security, Medicare, Additional Medicare, and California SDI components for two earners. The correct component amounts are $22,599, $6,235, $1,620, and $5,590, respectively." +us,scenario_005,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"The model used a $183,600 Social Security wage base instead of $184,500 and applied California SDI at 1.2% instead of 1.3%. Those parameter errors understate Social Security by $55.80 and SDI by $430." +us,scenario_005,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used an approximate 1.3% California SDI calculation and an incorrect Social Security wage-base amount, producing a $623 overstatement. Exact application of the $184,500 Social Security base and 1.3% SDI rate yields $36,044." +us,scenario_005,payroll_tax,gpt-5.6-sol,llm_error,thresholds_rates,False,"The answer understates the total by $430, which is exactly the difference between applying California SDI at 1.2% and 1.3% to $430,000. The required California SDI amount is $5,590, not $5,160." +us,scenario_005,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"The answer understates the total by $430, reflecting a 1.2% California SDI rate rather than the applicable 1.3% rate on $430,000. Correct SDI is $5,590." +us,scenario_005,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"The explanation lists only federal OASDI, Medicare, and Additional Medicare Tax and omits the mandatory California SDI component. California SDI adds $5,590 based on 1.3% of uncapped wages." +us,scenario_005,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly capped California SDI and reported only $4,226 instead of applying 1.3% to all $430,000 of wages for $5,590. It also used an incorrect Social Security cap for the spouse." +us,scenario_005,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly subtracted employer-sponsored insurance premiums from Social Security, Medicare, and Additional Medicare wage bases and imposed an obsolete per-worker cap on California SDI. The traced computation uses the full $430,000 wages and uncapped California SDI at 1.3%." +us,scenario_005,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model reduced FICA wages by employer-sponsored insurance premiums and capped California SDI wages. The correct calculation uses gross wages of $180,000 and $250,000 and applies uncapped 1.3% California SDI to their full $430,000 total." +us,scenario_005,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no payroll_tax value or explanation, so the required output was missing." +us,scenario_005,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,"The model correctly calculated the federal components but applied California SDI at 1.2%, producing $5,160 instead of $5,590. The applicable uncapped rate is 1.3%, accounting for the entire $430 shortfall." +us,scenario_005,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model incorrectly treated income-tax deductions and exemptions as offsets against employee payroll taxes. Social Security, Medicare, Additional Medicare Tax, and California SDI remain due on the wage bases and total $36,044." +us,scenario_005,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model substituted the 2025 Social Security wage base for the 2026 $184,500 base and imposed an obsolete California SDI wage cap at a 1.1% rate. For 2026, California SDI is uncapped at 1.3%, producing $5,590." +us,scenario_005,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model treated Medicare as capped by the Social Security wage base and failed to include the full California SDI contribution. Medicare applies at 1.45% to all $430,000, Additional Medicare adds $1,620, and uncapped California SDI adds $5,590." us,scenario_005,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model invented roughly $22,800 of mortgage-interest expense from an assumed 6.5% rate even though no mortgage interest was listed, then applied an unsupported high-income itemized-deduction reduction. The allowable deduction is $15,338.79, producing taxable income of $521,425.72 rather than its estimated $490,509." -us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model fabricated a deduction for federal income tax paid and then subtracted approximately $21,621 of nonexistent California nonrefundable credits. The applicable exemption credits total only $156.93 after the tax schedule produces $41,208.45." -us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model assumed about $60,000 of mortgage interest from the mortgage balance and an assumed interest rate, despite the instruction that unlisted expenses are zero. This inflated itemized deductions to about $79,854 instead of $15,338.79 and understated taxable income." -us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model added the $70,000 tax-exempt private pension to California income and invented about $35,000 of mortgage interest using an assumed rate. Those unsupported inputs replaced the traced AGI of $536,764.50 and deduction of $15,338.79 with an erroneous taxable-income estimate." -us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an estimated $560,000 taxable-income base and invoked rates rising to 11.3%, although $521,425.72 remains in the 9.3% California joint-filer bracket. Applying the actual schedule and $156.93 exemption credit yields $41,051.51, not $47,500." -us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model calculated deductions of $23,017 by treating the full $17,280 property-tax amount, all charity, and a self-computed miscellaneous deduction as independently deductible. The traced allowable deduction is $15,338.79, so taxable income is $521,425.72 rather than $511,253; it also used an estimated $500 exemption credit instead of $156.93." -us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented $50,000 of mortgage interest by applying an assumed 5% rate to the acquisition-debt limit. No mortgage-interest expense was supplied, and the resulting $69,854 itemized deduction improperly reduced taxable income to $464,416 instead of $521,425.72." -us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its stated taxable income of $497,700 is $23,725.72 below the traced $521,425.72, showing that it subtracted deductions or adjustments not allowed by the supplied facts. It also used a $300 exemption credit instead of the traced $156.93." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included an invented mortgage-interest deduction in $73,017 of itemized deductions. The allowable deduction is only $15,338.79, leaving taxable income of $521,425.72 rather than $461,253." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The unexplained $31,215 estimate does not reproduce any traced computation step. The required chain is $536,764.50 of California AGI, $15,338.79 of deductions, $41,208.45 of scheduled tax, and $156.93 of nonrefundable credits." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used taxable income of roughly $501,848 instead of $521,425.72. It therefore deducted about $19,578 more than the traced $15,338.79 allowable deduction before applying the California schedule." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model started from California AGI of $520,901 rather than $536,764.50 and deducted $19,854 rather than $15,338.79. These two income-base errors produced taxable income well below the traced $521,425.72." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated the listed property tax, charitable contributions, and employee expenses as additive California itemized deductions without applying the traced limitations. The resulting deduction differs from the allowable $15,338.79 and leads to an overstated tax." -us,scenario_005,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the $77,920 of qualified dividends from federal and California AGI, producing $449,439 instead of $536,764.50. It also incorrectly asserted a $2,000 California capital-loss limit and used an estimated standard deduction rather than the traced $15,338.79 deduction." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The $20,677 answer does not implement the California computation implied by the household's income. Taxable income is $521,425.72, whose scheduled tax is $41,208.45 before subtracting only $156.93 of nonrefundable credits." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The generic $37,099 estimate does not reproduce the traced income, deduction, rate-schedule, or credit amounts. Applying the schedule to $521,425.72 and subtracting $156.93 yields $41,051.51." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model says it reduced wages by both traditional 401(k) deferrals and pretax health premiums, but the traced California AGI is $536,764.50. Its additional income exclusions or excess itemized deductions lowered the tax below the $41,208.45 generated on $521,425.72 of taxable income." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used approximately $519,427 of taxable income instead of $521,425.72. That $1,998.72 understatement of the tax base, together with its unspecified exemption-credit calculation, produced the $27.51 shortfall." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,other,False,"The model supplied only a rounded estimate and did not execute the exact schedule-and-credit computation. The scheduled tax is $41,208.45 and the applicable nonrefundable credits are $156.93, producing $41,051.51 rather than $41,150." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model states that California taxes the tax-exempt interest, but the traced California AGI is $536,764.50 and does not support that blanket addition. Its income and deduction treatment therefore failed to reach taxable income of $521,425.72 before applying the schedule." -us,scenario_005,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The unexplained $48,720 answer exceeds the scheduled California tax of $41,208.45 even before credits. It failed to apply the joint-filer rate schedule to the traced $521,425.72 taxable-income base and subtract $156.93 of exemption credits." -us,scenario_005,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used California AGI of $534,270 and taxable income near $512,000 instead of $536,764.50 and $521,425.72. It also used an estimated $280 exemption credit rather than the traced $156.93." -us,scenario_005,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used incorrect 2026 joint-filer bracket thresholds and omitted all nonrefundable exemption credits. It also started from erroneous AGI of $525,049 and taxable income of $501,848 instead of $536,764.50 and $521,425.72." -us,scenario_005,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. This is a missing-output contract failure rather than a tax calculation. -us,scenario_005,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model understated California AGI by $2,494.50 and used $21,837 of deductions instead of $15,338.79, producing taxable income of about $512,433 rather than $521,425.72. It also used a $258 credit instead of $156.93." -us,scenario_005,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model wrongly concluded that retirement contributions, capital losses, and itemized deductions eliminated all California taxable income, and it incorrectly imported the federal $10,000 SALT cap. The household retains $521,425.72 of California taxable income and owes $41,051.51 after nonrefundable credits." -us,scenario_005,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model failed to limit the net capital loss to the allowable annual deduction when constructing AGI, then applied a rate schedule that generated only $33,544 on a claimed $565,375 of taxable income. It also used an unsupported $892 exemption credit rather than $156.93; the traced taxable income is $521,425.72 and scheduled tax is $41,208.45." +us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model invented roughly $22,800 of mortgage interest even though no mortgage-interest payment was listed, then combined it with an inaccurate high-income itemized-deduction reduction. The trace allows only $15,338.79 of California deductions and yields taxable income of $521,425.72, not $490,509." +us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted an estimated $23,000 of federal income tax and then invented $21,621 of California nonrefundable credits. California instead applies $15,338.79 of deductions and only $156.93 of exemption credits." +us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented about $60,000 of deductible mortgage interest from the loan balance and an assumed interest rate. That unsupported deduction drove taxable income down to $453,116 instead of the traced $521,425.72." +us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model added the $70,000 tax-exempt private pension to California income and simultaneously invented approximately $35,000 of mortgage interest. Those offsetting errors produced an unsupported taxable-income estimate rather than the traced $536,764.50 AGI and $521,425.72 taxable income." +us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an unsupported taxable-income estimate of about $560,000 based on unspecified mortgage interest and pension treatment. The traced taxable income is $521,425.72, which produces $41,208.45 before nonrefundable credits." +us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model overstated California itemized deductions at $23,017 by treating the full property tax, all charity, and a separately calculated employee-expense amount as deductible without reproducing California's limitation. The trace limits total deductions to $15,338.79, leaving taxable income of $521,425.72 rather than $511,253." +us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented $50,000 of mortgage interest from an assumed 5% rate and deducted $69,854 in total. No mortgage-interest payment was supplied, and the trace permits only $15,338.79 of deductions, so taxable income is $521,425.72 rather than $464,416." +us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used taxable income of $497,700 without deriving the traced California AGI or deductions. The correct computation leaves $521,425.72 taxable and subtracts $156.93, not $300, of exemption credits." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model claimed $73,017 of itemized deductions by including invented mortgage interest. The trace contains only $15,338.79 of deductions, producing taxable income of $521,425.72 rather than $461,253." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $31,215 implies a materially understated California tax base or misapplied schedule. The traced computation taxes $521,425.72, obtains $41,208.45 before credits, and subtracts only $156.93." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated taxable income at roughly $501,848. California deductions are $15,338.79 from AGI of $536,764.50, leaving $521,425.72 taxable." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used the wrong California AGI of $520,901 and overstated deductions at $19,854. The trace uses $536,764.50 of AGI and $15,338.79 of deductions, producing a higher $521,425.72 tax base." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated the listed real-estate tax, charity, and employee expenses as direct state itemized deductions without reproducing the traced limitation to $15,338.79. This overstated the liability because it did not derive the $521,425.72 taxable base and $156.93 exemption credit accurately." +us,scenario_005,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the $77,920 of qualified dividends from AGI and incorrectly asserted a $2,000 California capital-loss limit. Those errors reduced California AGI to $449,439 instead of $536,764.50." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The unexplained $20,677 is consistent with applying an effective or partial rate rather than California's progressive joint-filer schedule to the traced tax base. Tax on $521,425.72 is $41,208.45 before the $156.93 exemption credit." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The generic estimate did not derive California AGI, the $15,338.79 deduction, or the resulting $521,425.72 taxable income. Its $37,099 answer therefore understates the scheduled tax and exemption-credit result." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model reduced wages for both traditional 401(k) deferrals and pretax health premiums without establishing that the listed gross wages excluded neither amount, and it never quantified the traced deduction limitation. The engine reaches $536,764.50 of California AGI and $521,425.72 of taxable income." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used approximately $519,427 of taxable income instead of $521,425.72. That $1,998.72 base understatement, together with its imprecise credit calculation, accounts for the $27 shortfall." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model did not apply the exact 2026 California joint-filer schedule and $156.93 exemption credit to the traced taxable income. Those steps yield $41,208.45 before credits and $41,051.51 afterward, not $41,150." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly described tax-exempt interest as California-taxable and did not reconcile its income and deductions to the traced tax base. California taxable income is $521,425.72, followed by $156.93 of exemption credits." +us,scenario_005,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The unexplained $48,720 implies an overstated taxable base or rate. The traced base is $521,425.72, whose scheduled tax is $41,208.45 before subtracting $156.93." +us,scenario_005,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated AGI at $534,270 and taxable income at roughly $512,000, then overstated the surviving exemption credit at about $280. The trace uses $536,764.50 of AGI, $521,425.72 of taxable income, and a $156.93 credit." +us,scenario_005,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated California AGI at $525,049, overstated deductions at $23,201, and failed to subtract any nonrefundable exemption credit. The traced figures are $536,764.50 of AGI, $15,338.79 of deductions, and $156.93 of credits." +us,scenario_005,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly added both the $70,000 tax-exempt private pension and $8,961 tax-exempt interest to California AGI. This inflated AGI to about $604,010 instead of the traced $536,764.50." +us,scenario_005,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_005,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model understated California AGI at $534,270 and overstated deductions at $21,837, yielding taxable income of $512,433. The trace instead uses $536,764.50 of AGI and $15,338.79 of deductions, leaving $521,425.72 taxable." +us,scenario_005,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented enough mortgage interest and other deductions to eliminate tax and incorrectly invoked the federal $10,000 SALT cap in the California calculation. The actual California deduction is $15,338.79, leaving $521,425.72 taxable and substantial liability." +us,scenario_005,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model failed to subtract the traditional 401(k) deferrals when constructing income, yet its claimed tax on $565,375 is far below the California schedule's result and it overstated exemption credits at $892. The traced schedule produces $41,208.45 on $521,425.72 and permits only $156.93 of credits." +us,scenario_005,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model's stated taxable income of about $495,603 cannot produce only $29,423 under California's progressive joint-filer schedule. It also used a standard deduction rather than the traced $15,338.79 deduction, which leaves $521,425.72 taxable." us,scenario_005,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model discarded all SSDI from taxable income even after recognizing that Social Security taxability depends on combined income. The worksheet includes $13,784.12 of taxable SSDI, yielding $49,064.12 of AGI and $32,964.12 of taxable income after the $16,100 deduction." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly derived $13,784 of taxable Social Security and taxable income near $32,964, and its own bracket calculation produced about $3,713. It then replaced that result with $2,386 without any supporting computation." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the full 85% benefit cap, $16,392.25, as taxable Social Security instead of applying the worksheet amount of $13,784.12. Its final $1,858 also contradicts its own resulting tax calculation of about $4,021." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived $49,064 of AGI, $32,964 of taxable income, and approximately $3,708 of regular tax. It then submitted $2,534, a number unsupported by any deduction, credit, or calculation in its reasoning." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model equated exceeding the upper combined-income threshold with taxing a full 85% of SSDI, omitting the statutory Social Security worksheet calculation that yields $13,784.12. It also used the 2025 $15,000 standard deduction instead of the 2026 $16,100 amount." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model used an unsupported $15,650 taxable-Social-Security approximation instead of $13,784.12, but even its own inputs produced about $3,941 of tax. It then submitted $1,611 without applying any identified credit or other reduction." -us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $15,300 standard deduction instead of the 2026 single deduction of $16,100. This overstated taxable income by $800 and therefore overstated tax." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly included $13,784 of taxable SSDI but used an estimated $15,700 standard deduction instead of $16,100. It also substituted an incorrect lower-bracket cutoff, rather than applying the engine's 2026 main-rate parameters." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model treated this 56-year-old single taxpayer as a head-of-household filer over age 65 and invoked unidentified nonrefundable credits. The taxpayer is single, receives no age-based additional deduction, and has no applicable nonrefundable credits." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied pre-TCJA personal-exemption rules to 2026 instead of the applicable $16,100 single standard deduction and 2026 rate schedule. That obsolete deduction structure does not produce the traced taxable income of $32,964.12." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model substituted an $8,300 standard deduction plus a $5,150 personal exemption for the applicable $16,100 single standard deduction. This understated deductions by $2,650 and inflated taxable income and tax." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model derived AGI correctly but implicitly used a standard deduction smaller than $16,100. The correct deduction leaves $32,964.12 of taxable income, to which the 2026 main rates produce $3,707.70." -us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used 2024's $14,600 standard deduction and 2024 brackets for a 2026 calculation. It also overstated taxable Social Security at $14,025.53 instead of $13,784.12." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that the standard deduction offsets the taxable pension and taxable Social Security. Those items produce $49,064.12 of AGI and $32,964.12 of taxable income after the deduction, and no nonrefundable credit eliminates the resulting tax." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model invoked unspecified available nonrefundable credits to reduce the liability to zero, although the household has no applicable nonrefundable credits. After the $16,100 standard deduction, $32,964.12 remains taxable." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model correctly calculated AGI but used an estimated $15,750 standard deduction instead of $16,100. It also rounded away the exact result rather than applying the 2026 main-rate parameters to $32,964.12." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model overstated taxable income at about $35,572. Correctly including $13,784.12 of taxable Social Security and subtracting the $16,100 standard deduction yields $32,964.12." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model estimated taxable income at $33,206 instead of deriving $32,964.12 from $49,064.12 of AGI less the $16,100 standard deduction. Applying the exact 2026 brackets to the traced amount yields $3,707.70, not the rounded estimate." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model stated the correct components but did not use the exact 2026 deduction and main-rate parameters. The traced inputs produce $32,964.12 of taxable income and exactly $3,707.70 of tax." -us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The submitted amount reflects a substantial omission from taxable income, with no credit identified to explain the reduction. Taxable pension plus $13,784.12 of taxable SSDI leaves $32,964.12 taxable after the standard deduction." -us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly imposed projected post-TCJA-sunset deductions and 10%/15% brackets. The applicable 2026 computation uses the $16,100 standard deduction and PolicyEngine's 2026 main-rate schedule, with no personal exemption." -us,scenario_007,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model taxed the maximum 85% of SSDI, $16,392, instead of the Social Security worksheet result of $13,784.12. It also used a $15,420 standard deduction instead of $16,100, overstating taxable income from both directions." -us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. -us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly derived taxable Social Security and AGI but then used an unspecified deduction/exemption combination that did not equal the $16,100 single standard deduction. Applying that deduction and the 2026 main rates yields $3,707.70." -us,scenario_007,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model omitted the taxable portion of SSDI entirely and treated AGI as only the $35,280 pension. The Social Security worksheet adds $13,784.12 to AGI; the model also incorrectly used $1,540 as the tax on the first bracket rather than the applicable 10% amount." -us,scenario_007,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model misapplied the Social Security worksheet by using a $6,000 add-on in the upper-tier formula, producing $15,525.53 taxable SSDI instead of $13,784.12. It then used an estimated $15,750 standard deduction instead of $16,100." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model ultimately excluded all $13,784.12 of taxable Social Security disability benefits despite provisional income requiring that amount to be included. It also used a $14,600 standard deduction instead of $16,100, producing the wrong taxable-income base." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly derived approximately $49,064 of AGI, $32,964 of taxable income, and tax near $3,708, then submitted $2,386 without any supporting computation. Its final value contradicts its own correct derivation." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model treated the full 85% cap of $16,392 as taxable Social Security instead of applying the worksheet result of $13,784.12. It then submitted $1,858 even though its stated, already overstated tax calculation was about $4,021." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived $13,784 of taxable Social Security, $32,964 of taxable income, and approximately $3,708 of tax. It nevertheless submitted $2,534, a number unsupported by and contradictory to its own calculation." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly set taxable Social Security equal to the $16,392.25 maximum cap rather than the worksheet result of $13,784.12. It compounded that error by using estimated 2025 parameters instead of the 2026 $16,100 standard deduction and applicable brackets." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model used an unsupported taxable Social Security figure of $15,650 rather than $13,784.12 and calculated tax around $3,941. It then submitted $1,611 without any computation connecting that figure to taxable income or credits." +us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $15,300 standard deduction instead of the 2026 single deduction of $16,100. This overstated taxable income by $800 and therefore overstated regular income tax." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly included about $13,784 of taxable Social Security but used an estimated $15,700 standard deduction rather than $16,100. It also applied the wrong first-bracket threshold, yielding $3,765 instead of the tax on $32,964.12." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model treated the 56-year-old single taxpayer as a head-of-household filer over age 65 and invoked unspecified nonrefundable credits. The taxpayer is under 65, has no qualifying dependent, files single, and receives only the $16,100 single standard deduction." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied a pre-TCJA personal exemption in addition to its standard-deduction treatment. The 2026 computation uses the $16,100 standard deduction without a personal exemption, leaving $32,964.12 of taxable income." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model replaced the applicable $16,100 standard deduction with an $8,300 deduction plus a $5,150 personal exemption. Those obsolete components understated deductions by $2,650 and produced the wrong taxable income and tax." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model derived the correct AGI, so its $3,807.18 answer reflects application of the wrong deduction or bracket parameters. Using the $16,100 standard deduction gives $32,964.12 of taxable income, on which the 2026 brackets produce $3,707.70." +us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used 2024's $14,600 standard deduction and 2024 tax brackets for a 2026 calculation. It also overstated taxable Social Security as $14,025.53 instead of $13,784.12." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that the standard deduction offset the taxable pension and taxable Social Security. Those items produce $49,064.12 of AGI and $32,964.12 of taxable income after the deduction, so the liability is not zero." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the income as fully eliminated by the standard deduction and unspecified nonrefundable credits. The household has $32,964.12 of taxable income after the $16,100 deduction and no applicable nonrefundable credits." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model correctly estimated taxable Social Security and AGI but used a $15,750 standard deduction instead of $16,100. It then rounded an already incorrect bracket calculation to $3,750 rather than applying the exact 2026 parameters." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used approximately $35,572 of taxable income, overstating the correct $32,964.12 by $2,607.88. That figure is consistent with taxing the full 85% Social Security cap rather than the $13,784.12 worksheet amount." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model estimated taxable income as $33,206 instead of deriving $32,964.12 from $49,064.12 of AGI and the $16,100 deduction. The resulting approximate bracket calculation overstated the tax." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model stated the correct computation structure but did not apply the exact 2026 deduction and bracket parameters. The proper base is $32,964.12 of taxable income, yielding $3,707.70 rather than $3,736.66." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The submitted $2,444 reflects a substantially understated taxable-income base and provides no calculation of the taxable Social Security portion. Including $13,784.12 of Social Security produces $32,964.12 of taxable income after the standard deduction." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly projected a post-TCJA sunset regime with an $8,535 standard deduction, a $5,443 personal exemption, and a 15% bracket. The applicable 2026 computation instead uses the $16,100 standard deduction and current 10% and 12% brackets at this income." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model taxed the full $16,392 maximum of Social Security benefits rather than the worksheet amount of $13,784.12. It also used a $15,420 standard deduction instead of $16,100, overstating taxable income on both steps." +us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for the requested variable. +us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly identified taxable Social Security and AGI but applied an unspecified deduction/exemption regime that produced excessive tax. The applicable $16,100 standard deduction and 2026 brackets yield tax of $3,707.70." +us,scenario_007,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model omitted the entire $13,784.12 taxable portion of Social Security from AGI. It also miscomputed the 10% bracket tax as $1,540 on an $11,925 bracket segment, further distorting the liability." +us,scenario_007,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used the wrong Social Security worksheet expression, including $15,525.53 rather than $13,784.12 of benefits in AGI. It also used an estimated $15,750 standard deduction instead of $16,100." +us,scenario_007,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model's explanation omits taxable Social Security and derives only $2,177 of tax using an erroneous 12.5% rate, yet it submits $4,202. The final value has no connection to its stated computation; the correct calculation includes $13,784.12 of taxable Social Security and applies the 2026 10% and 12% brackets." us,scenario_007,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_007,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted constant full-year SSDI receipt into an assumption that the 24-month Medicare waiting period had already been completed. Constancy throughout 2026 establishes only the current year's status, so the 56-year-old head does not satisfy the age or stated disability-duration pathway." -us,scenario_007,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated a year-long SSDI status as proof that the beneficiary had completed Medicare's 24-month waiting period. No prior SSDI duration or other Medicare-qualifying condition was listed, and the head is under 65." -us,scenario_007,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,The model stated the correct 24-month rule but applied it without any fact showing 24 months of SSDI entitlement. The listed annual SSDI income and constant 2026 status do not establish completion of that waiting period. -us,scenario_007,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model invoked the disability qualifying period but assumed it was satisfied merely because SSDI income was present. The head is 56, and no completed 24-month SSDI period or alternative Medicare qualification was provided." -us,scenario_007,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model explicitly misread constant status throughout the tax year as evidence that the head had passed the 24-month SSDI waiting period. A single full year of stated SSDI receipt does not establish the required prior duration, so disability-based Medicare eligibility does not follow." -us,scenario_007,head_medicare_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model equated receipt of SSDI with immediate Medicare eligibility. For a 56-year-old, SSDI alone does not establish completion of the 24-month waiting period, and no alternative qualifying condition was listed." -us,scenario_007,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,The model treated SSDI recipient status as sufficient for Medicare eligibility. It omitted the disability waiting-period requirement and had no fact establishing another under-65 eligibility pathway. -us,scenario_007,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model incorrectly made SSDI receipt itself the qualifying event for Medicare. The head is under 65, and the facts do not establish completion of the 24-month SSDI waiting period or another qualifying condition." -us,scenario_007,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model applied disability-based Medicare eligibility directly from the SSDI payment. It skipped the required waiting-period step, which the annual income fact does not satisfy." -us,scenario_007,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model concluded that the $19,285 of SSDI automatically qualified the head under PolicyEngine rules. The amount of SSDI does not establish the required duration of entitlement, and the 56-year-old has no listed alternative Medicare qualification." -us,scenario_007,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,The model asserted that PolicyEngine sets Medicare eligibility from disability status and then inferred that status from SSDI income. It omitted the completed waiting-period or other qualifying-condition requirement for a person under 65. -us,scenario_007,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated SSDI income as sufficient support for the disability pathway. The facts establish neither completion of the 24-month SSDI waiting period nor another Medicare-qualifying condition, while age 56 fails the ordinary age test." -us,scenario_007,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,The model directly assigned disability-based Medicare eligibility from receipt of SSDI. It skipped the waiting-period determination and supplied no stated alternative basis for eligibility before age 65. -us,scenario_007,head_medicare_eligible,grok-build-0.1,llm_error,age_disability,False,The model incorrectly stated that SSDI income qualifies the head for Medicare under PolicyEngine rules. SSDI receipt without a completed 24-month entitlement period does not establish eligibility for this 56-year-old. -us,scenario_007,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,The model applied the under-65 disability pathway solely because SSDI income was listed. It failed to require evidence that the SSDI waiting period had elapsed or that another qualifying condition existed. -us,scenario_007,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model taxed nonwage income as if it were employee payroll-tax wages. The facts include Social Security disability income and private pension income, but no listed gross wages or salary; under the prompt's zero-default rule, hours worked without a wage amount create no payroll-tax base, so employee Social Security and Medicare tax is $0." -us,scenario_007,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable payroll_tax value. The correct computation uses no listed wage or salary income, so there is no employee Social Security or Medicare tax base and payroll_tax is $0." +us,scenario_007,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model explicitly assumed that the 24-month SSDI waiting period was satisfied, even though constant status during 2026 establishes only one full year and the prompt forbids inferring an unlisted prior receipt duration. At age 56 with no qualifying disability-duration input, the head is not Medicare eligible." +us,scenario_007,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated constant year-long SSDI receipt as completion of Medicare's 24-month waiting period. A full-year 2026 status does not supply the required prior SSDI duration, and the head is under 65." +us,scenario_007,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model cited the 24-month SSDI rule but applied it without any fact showing 24 months of entitlement. The prompt makes the unlisted qualifying duration zero, so the 56-year-old head does not qualify through disability or age." +us,scenario_007,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model converted receipt of Social Security disability income directly into Medicare eligibility without establishing completion of the qualifying waiting period. The income amount is not itself PolicyEngine's Medicare-eligibility status, and the head is only 56." +us,scenario_007,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model expressly inferred that constant SSDI receipt meant the head had passed the 24-month waiting period. Constancy applies only throughout the stated tax year and does not establish earlier receipt, while unlisted duration and disability-status inputs are false or zero." +us,scenario_007,head_medicare_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model treated the presence of Social Security disability income as conclusive Medicare eligibility. It omitted the disability-based qualifying conditions and waiting period, neither of which is supplied for this 56-year-old head." +us,scenario_007,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"The model equated SSDI receipt with immediate Medicare eligibility. SSDI income alone does not establish completion of the disability-based Medicare waiting period, and the head does not meet the age-65 rule." +us,scenario_007,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model treated Social Security Disability Insurance income as an automatic Medicare qualification. It failed to apply the required qualifying-duration condition, which is unlisted and therefore zero, for a person under age 65." +us,scenario_007,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model invoked disability-based Medicare eligibility solely from the SSDI payment. The prompt provides neither a completed 24-month entitlement period nor another qualifying condition, so the under-65 pathway does not apply." +us,scenario_007,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,The model collapsed the rule from eligible after qualifying SSDI entitlement into eligible whenever SSDI income is present. It omitted the waiting-period requirement and treated an income variable as the Medicare-eligibility determination. +us,scenario_007,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model claimed PolicyEngine eligibility was based on disability status, but the household facts do not list a disability-status input and explicitly make unlisted statuses false. SSDI income does not substitute for that status or establish the required entitlement duration." +us,scenario_007,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated SSDI income as sufficient support for the disability pathway. It failed to establish the qualifying waiting period or another under-65 Medicare condition, while the age pathway is unavailable at 56." +us,scenario_007,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model directly mapped Social Security disability income to the disability-based Medicare rule. The payment does not establish the required qualifying duration, and no qualifying disability or ESRD status is listed." +us,scenario_007,head_medicare_eligible,grok-build-0.1,llm_error,age_disability,False,"The model incorrectly stated that Social Security disability income itself qualifies the head under PolicyEngine rules. The eligibility calculation requires qualifying age or disability-related conditions, not merely a positive SSDI amount." +us,scenario_007,head_medicare_eligible,inkling,llm_error,age_disability,False,"The model cited Medicare eligibility after the 24-month waiting period but never established that the period had elapsed. With unlisted duration set to zero and age 56, neither the disability nor age pathway is satisfied." +us,scenario_007,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,The model assumed disability-based Medicare eligibility immediately from receipt of Social Security disability income. It omitted the qualifying-duration requirement and inferred an unlisted disability eligibility status contrary to the prompt. +us,scenario_007,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model treated SSDI receipt as automatically conferring Medicare eligibility. It failed to distinguish receipt of disability income from satisfaction of the under-65 Medicare conditions, including the qualifying waiting period." +us,scenario_007,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly recognized that no wage amount was listed, then contradicted that reasoning by inventing $1,516.30 of employee Social Security or Medicare tax on unspecified “applicable taxable wages.” The 50 weekly hours do not create wages when the unlisted wage rate and annual wage income are zero, and SSDI and private pensions do not enter the employee payroll-tax base." +us,scenario_007,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract." us,scenario_007,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reached $19,180 of Idaho taxable income but abandoned that calculation and submitted an unsupported $1,571 estimate. It failed to apply the 2026 schedule: subtract the $4,920 zero-rate bracket and tax the remaining $14,260 at 5.3%, yielding $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented a general Idaho pension exclusion and applied it to the head’s entire $35,280 private pension. This 56-year-old does not qualify for the claimed exclusion; after the Social Security subtraction and $16,100 standard deduction, Idaho taxable income is $19,180." -us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly derived approximately $19,180 of Idaho taxable income, then replaced its own tax computation with an unsupported $1,379 estimate. The 2026 schedule applies 5.3% only above the $4,920 zero-rate bracket, producing $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the Social Security disability benefit as entirely nontaxable federally and used the wrong standard deduction. Federal AGI includes $13,784.12 of taxable Social Security, Idaho subtracts that same amount, and the correct $16,100 deduction leaves $19,180 before applying the state schedule." -us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly identified $19,180 of Idaho taxable income and a roughly $4,700 zero-rate amount, but its $1,112 answer is inconsistent with those inputs. Subtracting the exact $4,920 zero-rate bracket and applying 5.3% gives $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $13,784.12 of federally taxable Social Security from Idaho AGI and instead introduced an inapplicable personal exemption. Idaho AGI is $35,280, the $16,100 standard deduction leaves $19,180, and the state schedule yields $755.78 before refundable credits." -us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model’s own pension-only approach put taxable income near the correct range, but it invented unspecified add-backs and a deduction phase-in to raise the liability to $1,800. Idaho instead uses a $16,100 standard deduction and taxes $19,180 only above the $4,920 zero-rate bracket at 5.3%." -us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $15,300 standard deduction instead of $16,100 and applied 5.8% to every dollar of taxable income. The correct taxable income is $19,180, of which only $14,260 above the $4,920 zero-rate bracket is taxed at 5.3%." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $15,700 deduction rather than $16,100 and applied a 5.8% flat rate from the first taxable dollar. Idaho’s 2026 schedule leaves $19,180 taxable and applies 5.3% only above $4,920, yielding $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer treated unspecified disability income as part of Idaho taxable income despite Idaho’s subtraction of the federally taxable Social Security amount. The trace leaves only $35,280 before the $16,100 standard deduction, and the resulting $19,180 enters the Idaho tax schedule." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The explanation recognized that Idaho excludes Social Security but did not apply the 2026 deduction and tax schedule that follow. The $35,280 Idaho AGI becomes $19,180 after the $16,100 standard deduction, and taxing only the amount above $4,920 at 5.3% gives $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model derived $21,830 by using the wrong deduction structure and adding a personal exemption rather than applying the $16,100 standard deduction. Idaho taxable income is $19,180, after which the $4,920 zero-rate bracket and 5.3% rate produce $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly began with $49,064.12 of federal AGI and subtracted $13,784.12 of taxable Social Security, but its final tax reflects the wrong deduction or rate treatment. The $16,100 standard deduction leaves $19,180, and Idaho taxes only the $14,260 above its zero-rate bracket at 5.3%." -us,scenario_007,state_income_tax_before_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly granted a disability-based retirement-benefit deduction that fully excluded the $35,280 private pension. The facts establish Social Security disability income, not eligibility of this private pension for that Idaho deduction, so $19,180 remains taxable after the standard deduction." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that exclusions and the standard deduction eliminate all Idaho taxable income. After the Social Security subtraction, $35,280 remains, and the $16,100 standard deduction leaves $19,180 rather than zero." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"A zero liability implies that the model erased the remaining pension income through unsupported deductions or credits. Idaho retains $35,280 after its Social Security subtraction, and the standard deduction leaves $19,180 subject to the state schedule." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a $5,000 zero-rate bracket instead of the 2026 Idaho amount of $4,920. With its otherwise correct $35,280 income, $16,100 standard deduction, and 5.3% rate, the exact computation is ($19,180 − $4,920) × 5.3% = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The submitted $1,885 does not incorporate Idaho’s $16,100 standard deduction and $4,920 zero-rate bracket. The trace leaves $19,180 of taxable income, and only $14,260 is taxed at 5.3%, producing $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly derived $19,180 of Idaho taxable income but effectively applied 5.3% to the entire amount. Idaho first assigns the initial $4,920 to the zero-rate bracket, so only $14,260 is taxed and the liability is $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The answer’s $1,759.89 is inconsistent with the federal-based income after Idaho’s Social Security subtraction and standard deduction. Those steps leave $19,180, not an amount capable of generating the submitted liability under Idaho’s 2026 schedule." -us,scenario_007,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model did not account for Idaho’s $4,920 zero-rate bracket after deducting the standard deduction from pension income. The resulting $19,180 is not all taxed at the positive rate; only $14,260 is taxed at 5.3%." -us,scenario_007,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an $8,535 standard deduction instead of the 2026 single-filer deduction of $16,100. The correct deduction leaves $19,180, and the subsequent zero-rate bracket reduces the amount taxed at 5.3% to $14,260." -us,scenario_007,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $13,784.12 of federally taxable Social Security when moving from federal AGI to Idaho AGI, which inflated taxable income to $36,252. Idaho AGI is $35,280 and the $16,100 standard deduction leaves $19,180." -us,scenario_007,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so the required structured answer was missing." -us,scenario_007,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used an incorrect deduction or exemption to reach $21,680 and an incorrect $2,500 zero-rate bracket. Idaho taxable income is $19,180 after the $16,100 standard deduction, and the correct zero-rate bracket is $4,920." -us,scenario_007,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely stated that Idaho has no individual income tax. Idaho imposes an individual income tax, and this household owes $755.78 under the 2026 brackets and rate." -us,scenario_007,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated all $19,285 of Social Security disability income as federally and state taxable instead of including only $13,784.12 federally and then subtracting that amount for Idaho. It also used the wrong standard deduction and rate structure; the correct Idaho taxable income is $19,180 and the tax is $755.78." -us,scenario_007,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,The model applied a $120 Idaho Grocery Credit amount instead of the 2026 base amount of $155. Age 56 does not reduce the applicable base credit below $155. -us,scenario_007,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,The model incorrectly stated that Idaho has no refundable individual income-tax credits. The Idaho Grocery Credit is refundable and awards this full-year qualifying resident $155. -us,scenario_007,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an obsolete $120 Grocery Credit rule and then submitted $140, contradicting its own computation. The applicable 2026 base amount for this qualifying resident is $155." -us,scenario_007,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model incorrectly excluded the Grocery Credit from refundable credits based on its filing mechanism. PolicyEngine classifies the qualifying Idaho Grocery Credit as refundable, producing $155." -us,scenario_007,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,The model misclassified the Idaho Grocery Credit as a nonrefundable or miscellaneous item. It belongs in state refundable credits and contributes $155 for this household. -us,scenario_007,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly made refundability depend on whether the credit exceeds Idaho tax liability and treated the credit as nonrefundable when liability is positive. The Grocery Credit remains a refundable credit, so the full $155 is reported in state refundable credits." -us,scenario_007,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,The model incorrectly treated Idaho’s Grocery Credit as nonrefundable in this calculation. The full-year qualifying resident receives a $155 refundable Grocery Credit. -us,scenario_007,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model identified the correct credit but used $120 instead of Idaho’s 2026 base amount of $155. With one qualifying resident for all 12 months, the result is $155." -us,scenario_007,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model applied a $120 per-person Grocery Credit amount rather than the 2026 amount of $155. The single qualifying household member generates $155. -us,scenario_007,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model denied all refundable-credit eligibility despite the head being a full-year qualifying Idaho resident. That eligibility pathway awards the $155 Idaho Grocery Credit. -us,scenario_007,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly identified Grocery Credit eligibility but used an outdated $120 amount. The applicable 2026 base credit is $155. -us,scenario_007,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used a $120 Grocery Credit amount for the under-65 resident instead of the applicable 2026 base amount of $155. Full-year qualification yields the entire $155. -us,scenario_007,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model identified the refundable Grocery Credit but assigned it a $120 per-resident value. Idaho’s applicable 2026 base amount is $155. -us,scenario_007,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model falsely treated the Idaho Grocery Credit as repealed and therefore omitted it. The operative 2026 credit awards this qualifying resident $155, independently of the model’s assertion about state tax liability." -us,scenario_007,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model overlooked the full-year Idaho residency facts that qualify the head for the Grocery Credit. Applying that credit produces $155 in refundable state credits. -us,scenario_007,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model incorrectly required additional qualifying conditions beyond the stated Idaho residency and absence of disqualifying factors. The head qualifies for all 12 months and receives the $155 Grocery Credit. -us,scenario_007,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model invented an income phaseout that eliminated the Idaho Grocery Credit. This household’s stated income does not remove the traced eligibility, and the full refundable amount is $155." -us,scenario_007,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The model omitted Idaho’s refundable Grocery Credit entirely. The full-year qualifying resident receives $155 under that program. -us,scenario_007,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model used a $100 Grocery Credit amount instead of the applicable 2026 base amount of $155. Full-year residency entitles the qualifying person to the full $155. -us,scenario_007,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model failed to apply the Idaho Grocery Credit eligibility pathway to the full-year resident. That pathway supplies $155 in refundable state credits. -us,scenario_007,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no state_refundable_credits output. The required parsed value was $155 from the full-year Idaho Grocery Credit. -us,scenario_007,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model correctly selected the Grocery Credit but used $120 rather than the 2026 base amount of $155. No other refundable credits are needed to reach the $155 result. -us,scenario_007,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model incorrectly asserted that Idaho has no state income tax and used that assertion to discard refundable credits. Idaho’s Grocery Credit applies to this resident and contributes $155. -us,scenario_007,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model overlooked the applicable refundable Idaho Grocery Credit. The household’s full-year qualification produces a $155 refundable state credit. -us,scenario_008,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied only CHIP's under-19 age rule and upper income threshold. It omitted the prerequisite that the child not qualify for Medicaid; this 17-year-old qualifies for Medicaid in New Jersey's OLDER_CHILD category, which precludes CHIP." -us,scenario_008,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated age under 19 and income below roughly 350% FPL as sufficient for CHIP. It failed to evaluate Medicaid first: the child qualifies through New Jersey's OLDER_CHILD category and therefore cannot qualify for CHIP. -us,scenario_008,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model calculated against the CHIP ceiling and treated the absence of listed coverage as establishing CHIP availability. Current coverage is irrelevant to the decisive coordination rule: Medicaid eligibility through the OLDER_CHILD category excludes this child from CHIP. -us,scenario_008,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used the shortcut that a child in a low-income household qualifies for CHIP. Low income instead places this 17-year-old in New Jersey's Medicaid-eligible OLDER_CHILD category, and Medicaid eligibility precludes CHIP." -us,scenario_008,child1_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the under-19 age range and income below New Jersey's CHIP limit as sufficient conditions. It skipped the Medicaid screen, under which the child qualifies as an OLDER_CHILD and is consequently ineligible for CHIP." -us,scenario_008,child1_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility directly from age 17 and low household income. It failed to apply Medicaid precedence: New Jersey's OLDER_CHILD pathway makes the child Medicaid eligible, automatically excluding CHIP eligibility." -us,scenario_008,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring an explicitly listed special trigger and failed to apply the ordinary child MAGI pathway. Child 1 is a 17-year-old dependent in New Jersey, so the OLDER_CHILD Medicaid category applies, and MAGI at 0.55 times FPL satisfies that category's income test." -us,scenario_008,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 17 as within the WIC child age range. WIC child eligibility for children ends before age 5, so Child 1 fails the categorical age test regardless of the household's other facts." -us,scenario_008,child2_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied New Jersey's CHIP age and 355% FPL ceiling as sufficient conditions and omitted the exclusion for children already eligible for Medicaid. Child 2 qualifies for Medicaid under the OLDER_CHILD category, which disqualifies the child from CHIP." -us,scenario_008,child2_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated income below New Jersey FamilyCare's CHIP ceiling as establishing CHIP eligibility without first checking Medicaid eligibility. Child 2 is Medicaid-eligible under the OLDER_CHILD category and therefore is not CHIP-eligible. -us,scenario_008,child2_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model checked age, household income, and absence of listed private insurance but omitted the controlling Medicaid-exclusion step. Child 2's OLDER_CHILD Medicaid eligibility bars CHIP coverage even though the child is under 19 and household income is below the CHIP ceiling." -us,scenario_008,child2_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility directly from the child's age and low household income, skipping the required check that the child does not qualify for Medicaid. Child 2 qualifies for Medicaid under the OLDER_CHILD category, so CHIP eligibility is false." -us,scenario_008,child2_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model treated being under 19 and below New Jersey's CHIP income limit as sufficient for eligibility. It failed to apply the rule excluding Medicaid-eligible children; Child 2 is eligible for Medicaid under the OLDER_CHILD category. -us,scenario_008,child2_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model used the shortcut that a low-income 14-year-old in New Jersey qualifies for CHIP and omitted Medicaid coordination. Child 2 qualifies for Medicaid under the OLDER_CHILD category, which makes the child ineligible for CHIP." -us,scenario_008,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring an explicitly listed benchmark trigger and ignored the standard child MAGI pathway. A 14-year-old dependent in New Jersey is in the older-child Medicaid category, and household MAGI at 0.55 times FPL satisfies that category, so child2 is eligible." -us,scenario_008,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a WIC child age range of 1-18 and treated a 14-year-old as categorically eligible. It also reversed the income comparison by saying $30,915 was at or below the 185% FPL WIC limit when PolicyEngine's threshold for this New Jersey household is lower, so both the age gate and income gate fail." -us,scenario_008,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 14 as within the WIC child age range and stopped there. WIC child eligibility does not cover school-age children or teenagers; only infants and children under age 5 can qualify, so Child 2 fails the categorical age requirement." -us,scenario_008,child3_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated low income and housing-assistance receipt as affirmative CHIP eligibility without first testing Medicaid eligibility. Child 3 qualifies for Medicaid under New Jersey's OLDER_CHILD category, and that Medicaid eligibility makes the child ineligible for CHIP." -us,scenario_008,child3_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied only NJ FamilyCare's CHIP upper income limit of 355% FPL. It omitted the mutually exclusive Medicaid screen: this income places child 3 in the OLDER_CHILD Medicaid category, which precludes CHIP eligibility." -us,scenario_008,child3_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility solely because income was below the CHIP ceiling. Child 3 instead qualifies for Medicaid under the OLDER_CHILD pathway, and children eligible for Medicaid are not eligible for CHIP." -us,scenario_008,child3_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model compared household income only with the 355% FPL CHIP maximum and skipped the lower-income Medicaid pathway. At this income, the 12-year-old qualifies for Medicaid as an OLDER_CHILD, so CHIP eligibility is barred." -us,scenario_008,child3_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated age, lack of listed coverage, and income below CHIP's ceiling as sufficient. It failed to apply the prior Medicaid-eligibility exclusion: child 3 is Medicaid-eligible under the OLDER_CHILD category and therefore not CHIP-eligible." -us,scenario_008,child3_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used low income as a shortcut to CHIP eligibility. The correct categorical sequence assigns child 3 to Medicaid's OLDER_CHILD category at this income, and Medicaid eligibility precludes CHIP." -us,scenario_008,child3_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model equated being a low-income New Jersey child with qualifying for CHIP. Low income instead makes this 12-year-old Medicaid-eligible under the OLDER_CHILD category, which excludes the child from CHIP." -us,scenario_008,child3_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model applied only CHIP's 355% FPL ceiling and never screened for Medicaid eligibility; it also incorrectly added an American Opportunity Credit amount to MAGI even though tax credits are not income. Under the applicable income calculation, child 3 qualifies for Medicaid in the OLDER_CHILD category, and that eligibility precludes CHIP." -us,scenario_008,child3_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring an explicit benchmark-triggering fact instead of applying the New Jersey child MAGI pathway. Child3 is a 12-year-old dependent in the older-child Medicaid category, and the household MAGI of 0.55 FPL is below the applicable income threshold, so the correct eligibility result is yes." -us,scenario_008,child3_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated child age as the only WIC eligibility condition and omitted the WIC household income screen. Child 3 is age 12, but the household income of about $30,915 exceeds New Jersey's 2026 WIC limit at 185% of FPL for the household size, so the age condition does not create eligibility." -us,scenario_008,child4_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated New Jersey's approximately 355% FPL CHIP ceiling as a standalone eligibility test. It omitted the prior Medicaid determination: child4 qualifies under the OLDER_CHILD category, which bars CHIP eligibility." -us,scenario_008,child4_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model checked age and the CHIP upper-income limit but failed to test whether child4 qualified for Medicaid first. The household's low income makes child4 Medicaid-eligible under the OLDER_CHILD category, and that eligibility makes CHIP unavailable." -us,scenario_008,child4_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly concluded that being under 19 and below 355% FPL establishes CHIP eligibility. At this much lower income, child4 falls into New Jersey's Medicaid OLDER_CHILD category, and Medicaid eligibility excludes the child from CHIP." -us,scenario_008,child4_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model applied only NJ FamilyCare's CHIP maximum-income threshold and interpreted lower income as qualifying. It skipped the Medicaid pathway that covers child4 as an OLDER_CHILD and therefore makes CHIP eligibility false. -us,scenario_008,child4_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used low household income as direct support for CHIP without applying Medicaid's older-child eligibility pathway. Child4's low income instead establishes Medicaid eligibility, which disqualifies the child from CHIP." -us,scenario_008,child4_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model equated a low-income New Jersey child with CHIP eligibility and omitted the program-ordering rule. Child4 qualifies for Medicaid as an OLDER_CHILD, so the mutually exclusive CHIP test returns false." -us,scenario_008,child4_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,The model compared MAGI only with CHIP's 355% FPL ceiling and treated the absence of other coverage as sufficient. It failed to apply the lower-income Medicaid OLDER_CHILD pathway; child4's Medicaid eligibility itself precludes CHIP. -us,scenario_008,child4_head_start_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated child 4's age 10 status as sufficient for Head Start eligibility and ignored that this output is specifically Head Start for preschool-age children, not Early Head Start or school-age services. It also failed to apply the household income comparison against the 2026 poverty-guideline threshold, which PolicyEngine used to return not eligible." -us,scenario_008,child4_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored New Jersey's older-child Medicaid pathway for a 10-year-old dependent and treated eligibility as requiring an explicitly listed Medicaid-enrollment or trigger fact. Applying the MAGI child category to the household gives 0.55 x FPL, below the applicable threshold, so child4 is Medicaid eligible." -us,scenario_008,child4_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 10 as sufficient for WIC eligibility and skipped the household income test. Under the PolicyEngine calculation, the household's approximately $30,915 income exceeds the WIC income limit, so child 4 is not WIC-eligible." -us,scenario_008,child4_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model used a school-age categorical shortcut and never applied the WIC household income ceiling. PolicyEngine's derivation disqualifies child 4 because household income is above the applicable WIC limit, so the correct binary value is 0." -us,scenario_008,child5_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the CHIP income ceiling directly and failed to test Medicaid eligibility first. Child 5 qualifies for Medicaid under the YOUNG_CHILD category, so CHIP eligibility is false regardless of being below the CHIP ceiling; housing assistance does not create a separate CHIP pathway." -us,scenario_008,child5_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Child 5 was outside the Medicaid-only age-and-income category and assigned CHIP based on the roughly 355% FPL ceiling. At age 5 and this income, the child qualifies for Medicaid under YOUNG_CHILD, which disqualifies the child from CHIP." -us,scenario_008,child5_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated income below New Jersey's CHIP limit as sufficient for eligibility and omitted the required Medicaid-ineligibility test. Child 5's age and household income establish Medicaid eligibility under YOUNG_CHILD, so the child cannot qualify for CHIP." -us,scenario_008,child5_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model compared income only with the 355% FPL CHIP ceiling and treated lack of other coverage as dispositive. It failed to apply the lower-income Medicaid pathway: Child 5 qualifies under YOUNG_CHILD, and Medicaid eligibility precludes CHIP." -us,scenario_008,child5_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated combined NJ FamilyCare coverage with the distinct CHIP eligibility output by saying the child qualifies for 'CHIP/Medicaid.' PolicyEngine assigns this low-income 5-year-old to Medicaid's YOUNG_CHILD category, and a Medicaid-eligible child is not CHIP-eligible." -us,scenario_008,child5_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used low income as a shortcut to CHIP eligibility without distinguishing Medicaid from CHIP. The low income instead places Child 5 in Medicaid's YOUNG_CHILD category, which makes CHIP eligibility false." -us,scenario_008,child5_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility directly from age and low household income. Those facts establish Medicaid eligibility under YOUNG_CHILD, and CHIP excludes children who qualify for Medicaid." -us,scenario_008,child5_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model compared its MAGI estimate only with New Jersey's CHIP ceiling and considered lack of other coverage, but omitted the prerequisite that the child not qualify for Medicaid. At age 5 and this income, Child 5 is Medicaid-eligible under YOUNG_CHILD and therefore ineligible for CHIP." -us,scenario_008,child5_early_head_start_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model used a birth-through-age-5 age range for Early Head Start, merging Early Head Start with preschool Head Start. Child 5 is age 5, so the Early Head Start under-age-3 requirement fails even though income and housing-assistance facts support other low-income program pathways." -us,scenario_008,child5_head_start_eligible,claude-sonnet-4.6,llm_error,age_disability,False,The model imposed a hard ages-3-to-4 limit on Head Start and excluded the child solely because the child is age 5. It missed that PolicyEngine's Head Start preschool-age output still evaluates this 5-year-old and that the household qualifies through SNAP and income below 130% of the federal poverty guideline. -us,scenario_008,child5_head_start_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model used the usual preschool age range as a categorical exclusion for an age-5 child. It did not apply the benchmark's eligibility computation, which counts the 5-year-old in the Head Start output and finds eligibility through SNAP categorical eligibility and the direct income test." -us,scenario_008,child5_head_start_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,The model treated age 5 as aging the child out of standard Head Start and stopped at that shortcut. It missed the PolicyEngine rule path that evaluates the 5-year-old for Head Start and grants eligibility because the household receives SNAP and has adjusted gross income below 130% of the federal poverty guideline. -us,scenario_008,child5_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring a specific prompt fact that directly triggers enrollment or receipt, instead of applying New Jersey's child Medicaid categorical pathway. For a 5-year-old dependent, the engine uses the YOUNG_CHILD MAGI category, and the household's 0.55 FPL MAGI is below that category's income threshold, so child5 is eligible." -us,scenario_008,child5_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model treated age five as within WIC’s child category even though eligibility ends at the fifth birthday. It also incorrectly used housing-assistance receipt as evidence of WIC eligibility. -us,scenario_008,child5_wic_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model included a five-year-old in the WIC child category, which is limited to children under five. It also invented SNAP receipt even though the household facts do not list SNAP benefits." -us,scenario_008,child5_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model contradicted its own statement that eligibility lasts only until the fifth birthday by treating a child currently age five as eligible. Housing assistance is not a WIC adjunctive-eligibility program and cannot cure the age failure. -us,scenario_008,child5_wic_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The answer directly treated age five as WIC-eligible, omitting the rule that the child category ends at the fifth birthday." -us,scenario_008,child5_wic_eligible,glm-5.2,llm_error,categorical_eligibility,False,The model interpreted “eligibility ends at age five” as making age five qualifying. A child with annual age five has already reached the fifth birthday and is outside WIC’s child category. -us,scenario_008,child5_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model placed a five-year-old inside the benchmark’s WIC child age range. That range requires the child to be under five. -us,scenario_008,child5_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model assumed that age five belongs to the WIC-eligible group without applying the under-five cutoff. Reaching age five ends categorical eligibility as a child. -us,scenario_008,child5_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,The model declared age five qualifying without applying WIC’s fifth-birthday cutoff. Child 5 is outside the eligible child category. -us,scenario_008,child5_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated age five as categorically eligible, but WIC’s child category ends at the fifth birthday. Its household-level poverty comparison cannot override that categorical age failure." -us,scenario_008,child6_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied New Jersey's CHIP income ceiling directly and omitted the prior Medicaid determination. At age one and this household income, the child qualifies for Medicaid under the YOUNG_CHILD category, and that Medicaid eligibility makes the child ineligible for CHIP." -us,scenario_008,child6_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated being under the CHIP age and income limits as sufficient, without testing whether the child already qualified for Medicaid. The one-year-old qualifies through New Jersey's YOUNG_CHILD Medicaid category, which precludes CHIP eligibility." -us,scenario_008,child6_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model compared income with the CHIP limit for infants but skipped Medicaid eligibility. The child's age and low household income place the child in the YOUNG_CHILD Medicaid category, so CHIP's requirement that the child not qualify for Medicaid fails." -us,scenario_008,child6_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly listed age, CHIP income, and lack of other health coverage as all relevant conditions while omitting the Medicaid-exclusion condition. The child is Medicaid-eligible under the YOUNG_CHILD category, and existing Medicaid eligibility—not other insurance enrollment—bars CHIP." -us,scenario_008,child6_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated the combined NJ FamilyCare umbrella with CHIP eligibility and used the high CHIP ceiling as the decisive test. The child's low income and age instead establish Medicaid eligibility under YOUNG_CHILD, which makes the child ineligible for the CHIP component." -us,scenario_008,child6_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used the shortcut that a low-income child qualifies for CHIP, ignoring the Medicaid-first coverage pathway. This one-year-old qualifies for Medicaid under YOUNG_CHILD, and CHIP excludes children who qualify for Medicaid." -us,scenario_008,child6_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model treated household income below New Jersey's CHIP ceiling as sufficient for CHIP eligibility. It failed to apply the YOUNG_CHILD Medicaid category first; that category makes the child Medicaid-eligible and therefore CHIP-ineligible. -us,scenario_008,child6_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model equated young age and low income with CHIP eligibility without checking Medicaid. Those facts qualify the one-year-old for Medicaid under New Jersey's YOUNG_CHILD category, which disqualifies the child from CHIP." -us,scenario_008,child6_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model relied on housing assistance, low income, and absence of listed health coverage while omitting the Medicaid-exclusion rule. The child qualifies for Medicaid under YOUNG_CHILD, and Medicaid eligibility itself prevents CHIP eligibility regardless of other coverage." -us,scenario_008,child6_early_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model inverted the Early Head Start age rule: age 1 is inside the eligible infant/toddler range, not a disqualifying age. It therefore stopped at a false age exclusion and never applied the household-income test showing income below the 100% federal poverty line threshold." -us,scenario_008,child6_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated a 1-year-old as within the preschool-age Head Start category, collapsing Head Start and Early Head Start age rules. It also stopped at age and did not apply the household income eligibility screen that PolicyEngine used, under which the eight-person household's approximately $30,915 income exceeds the applicable Head Start cutoff." -us,scenario_008,child6_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring a specifically listed trigger such as current Medicaid receipt, SSI, disability, or another explicit status fact. It missed the age-based YOUNG_CHILD MAGI pathway for a 1-year-old in New Jersey and failed to apply the household's 0.55 FPL MAGI income level to that category." +us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reached $19,180 of Idaho taxable income, then abandoned its own $1,092 flat-rate calculation and asserted $1,571 without a supporting computation. Applying Idaho's 2026 brackets to $19,180 yields $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model wrongly treated the entire $35,280 private pension as qualifying for Idaho's retirement-benefits deduction. A 56-year-old does not receive a blanket Idaho exclusion for arbitrary qualified-plan pension income, so $19,180 remains taxable after the Social Security subtraction and standard deduction." +us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly derived approximately $13,785 of taxable Social Security and $19,180 of Idaho taxable income, but replaced Idaho's 2026 bracket calculation with unsupported rate estimates and then asserted $1,379. The applicable brackets produce $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded all SSDI from federal AGI and used an estimated $15,750 standard deduction instead of $16,100. It then subtracted a fabricated $4,673 threshold and discarded its own $846 result in favor of an unsupported $1,300." +us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model reached the correct $19,180 taxable-income base but misapplied a 5.3% flat rate and an additional roughly $4,700 exemption. Idaho's 2026 brackets applied to $19,180 yield $755.78, not $1,112." +us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $13,784.12 of federally taxable Social Security when computing Idaho AGI and instead invented a personal exemption and grocery-credit treatment. Idaho AGI is $35,280, taxable income is $19,180 after the $16,100 standard deduction, and the bracket tax is $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used an estimated $14,600 deduction and a flat 5.695% calculation, then raised its own computed $1,178 to $1,800 based on unspecified add-backs and phase-ins. The correct $16,100 deduction leaves $19,180, whose bracket tax is $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $15,300 standard deduction instead of $16,100, overstating taxable income by $800. It also applied 5.8% uniformly rather than the 2026 Idaho bracket computation that yields $755.78 on $19,180." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model understated the standard deduction by $400 and applied a 5.8% flat rate to the entire resulting base. The $16,100 deduction leaves $19,180, and Idaho's brackets produce $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer provides no calculation and its $1,582 result does not reflect the traced deductions. Federal AGI of $49,064.12 is reduced by $13,784.12 of taxable Social Security and the $16,100 standard deduction before Idaho's brackets produce $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model recognized Idaho's Social Security exclusion but did not apply the traced $16,100 standard deduction and 2026 bracket schedule correctly. Those steps leave $19,180 taxable and produce $755.78, not $1,246." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model calculated $21,830 of taxable income by using the wrong deduction structure, including an unsupported personal-exemption treatment. Idaho taxable income is $19,180 after the $16,100 standard deduction, and the model also wrongly treated 5.695% as a flat rate on the whole base." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly derived Idaho AGI of $35,280 but applied the wrong tax-rate computation after the standard deduction. The $16,100 deduction leaves $19,180, on which Idaho's 2026 brackets yield $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model invented a disability-based retirement-benefit deduction that fully excluded the $35,280 private pension. The pension remains in Idaho AGI; after the Social Security subtraction and $16,100 standard deduction, $19,180 is taxable." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that exclusions and the standard deduction fully offset the income. The $35,280 Idaho AGI exceeds the $16,100 standard deduction by $19,180, producing $755.78 of tax." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The zero answer implies that the model erased the $19,180 taxable balance or applied unsupported nonrefundable credits. Idaho AGI is $35,280 after the Social Security subtraction, and the $16,100 standard deduction does not eliminate it." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly reached income of $35,280 before the standard deduction but invented a separate $5,000 zero bracket and applied 5.3% only above it. Idaho's actual 2026 brackets applied to the $19,180 taxable base yield $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The unsupported $1,885 estimate does not incorporate the traced Idaho subtraction and deduction correctly. Subtracting $13,784.12 of taxable Social Security from federal AGI and then the $16,100 standard deduction leaves $19,180 for the bracket calculation." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly identified $19,180 of Idaho taxable income but used the wrong rate computation to obtain $1,017. Idaho's 2026 bracket schedule on that base yields $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $1,759.89 estimate does not account correctly for the Idaho subtraction of federally taxable Social Security and the $16,100 standard deduction. Those adjustments reduce the base to $19,180 before the brackets yield $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave no specific taxable-income, deduction, bracket, or credit calculation supporting $1,477. The pension-derived Idaho AGI is reduced by the $16,100 standard deduction to $19,180, and the applicable brackets yield $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a projected $8,535 standard deduction instead of $16,100, overstating taxable income by $7,565. It then applied 5.695% as a flat rate rather than using Idaho's 2026 brackets." +us,scenario_007,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model retained federally taxable Social Security in Idaho income, producing $36,252 after its estimated deduction instead of subtracting $13,784.12 first. Idaho taxable income is $19,180, and the model also wrongly applied a uniform 5.8% rate." +us,scenario_007,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an unexplained $32,964 federal taxable-income base rather than Idaho's $19,180 taxable income after the Social Security subtraction and $16,100 standard deduction. It also applied 5.3% uniformly instead of the applicable bracket schedule." +us,scenario_007,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response failed the required output contract." +us,scenario_007,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model left $21,680 taxable by using the wrong deduction or exemption amount, then invented a $2,500 zero bracket and applied 5.3%. The $16,100 standard deduction leaves $19,180, and Idaho's actual brackets yield $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely stated that Idaho has no individual income tax. Idaho taxes this household's $19,180 of taxable income under its 2026 state brackets, producing $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated all $19,285 of SSDI as federally and Idaho-taxable, rather than including $13,784.12 federally and subtracting that amount for Idaho. It also used the wrong standard deduction, flat rate, and unsupported credits, whereas the traced $19,180 base yields $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model double-counted deductions by starting from a purported $19,530 federal taxable base and subtracting another $5,200 Idaho standard deduction. It then used fabricated bracket amounts; the correct single $16,100 deduction leaves $19,180 and the actual brackets yield $755.78." +us,scenario_007,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,The model applied an obsolete $120 Grocery Credit amount instead of Idaho’s 2026 base amount of $155. Age 56 does not reduce the applicable 2026 base credit below $155. +us,scenario_007,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly stated that Idaho has no refundable individual income-tax credit. The Idaho Grocery Credit is refundable, and this full-year resident receives its $155 base amount." +us,scenario_007,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model identified the Grocery Credit but used inconsistent obsolete amounts, reasoning from $120 and submitting $140. Idaho’s applicable 2026 base amount for this resident is $155." +us,scenario_007,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model incorrectly excluded the Grocery Credit from refundable credits based on how it can be claimed. Its refundable character does not disappear when the taxpayer files an income-tax return, and the qualifying annual amount is $155." +us,scenario_007,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,The model misclassified Idaho’s Grocery Credit as a nonrefundable or miscellaneous item. PolicyEngine includes the qualifying $155 Grocery Credit in state refundable credits. +us,scenario_007,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly made refundability depend on whether the taxpayer has sufficient Idaho tax liability and treated the credit as nonrefundable when used against that liability. The Grocery Credit remains a refundable credit, and SSDI receipt does not remove the full $155 amount." +us,scenario_007,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,The model incorrectly classified Idaho’s Grocery Credit as nonrefundable. This full-year Idaho resident qualifies for a $155 refundable Grocery Credit. +us,scenario_007,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,The model used an obsolete $120 Grocery Credit parameter. The 2026 base amount applied to this resident is $155. +us,scenario_007,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model correctly identified the refundable Grocery Credit but used $120 per qualifying resident. Idaho’s applicable 2026 amount is $155. +us,scenario_007,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model omitted the full-year resident’s eligibility for the Idaho Grocery Credit. Twelve qualifying months produce a refundable credit of $155. +us,scenario_007,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model used an obsolete $120 amount after correctly identifying Grocery Credit eligibility. The resident’s age and lack of SNAP receipt lead to the applicable 2026 base credit of $155. +us,scenario_007,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model applied a $120 Grocery Credit parameter for a resident under age 65. The applicable 2026 base amount is $155. +us,scenario_007,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model correctly treated the Grocery Credit as refundable but used the wrong annual amount of $120. The 2026 base credit is $155. +us,scenario_007,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model falsely treated Idaho’s Grocery Credit as repealed. The credit applies in 2026 and awards this qualifying full-year resident $155, regardless of the model’s separate assertion about state tax liability." +us,scenario_007,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model overlooked the Idaho Grocery Credit triggered by full-year residency. The household qualifies for all 12 months and receives $155. +us,scenario_007,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model required additional qualifying facts even though Idaho residency and the stated absence of disqualifying facts establish Grocery Credit eligibility. The full-year credit is $155. +us,scenario_007,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,The model invented an income phaseout that disqualified the household from the Idaho Grocery Credit. The stated pension and disability income do not phase out the applicable $155 credit. +us,scenario_007,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model omitted the refundable Idaho Grocery Credit applicable to this full-year resident. The qualifying annual amount is $155. +us,scenario_007,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model used an obsolete $100 Grocery Credit amount. Idaho’s applicable 2026 refundable base amount is $155. +us,scenario_007,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model failed to apply the Idaho Grocery Credit eligibility pathway for a full-year resident with no disqualifying facts. Twelve qualifying months yield $155. +us,scenario_007,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no state_refundable_credits output. The required output was the $155 Idaho Grocery Credit. +us,scenario_007,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model correctly identified the sole applicable refundable credit but used an obsolete $120 Grocery Credit amount. The 2026 base amount is $155. +us,scenario_007,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model incorrectly asserted that Idaho has no state income tax and consequently omitted its refundable credits. Idaho imposes an individual income tax and provides this resident a refundable Grocery Credit of $155. +us,scenario_007,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model overlooked the Idaho Grocery Credit applicable to the household’s circumstances. Full-year qualification produces a $155 refundable state credit. +us,scenario_007,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model failed to apply the Idaho Grocery Credit to this full-year resident. With all 12 months qualifying and no disqualifying factor, the refundable credit is $155." +us,scenario_008,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied only CHIP's age and upper-income conditions and omitted the requirement that the child not qualify for Medicaid. Child 1 is Medicaid-eligible in New Jersey's OLDER_CHILD category, which precludes CHIP regardless of being below the CHIP income limit." +us,scenario_008,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated low income and age under 19 as sufficient for CHIP without first testing Medicaid eligibility. Child 1 qualifies for Medicaid as an OLDER_CHILD, so CHIP eligibility is automatically denied." +us,scenario_008,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model compared income directly with the 350% FPL CHIP ceiling and treated lack of listed coverage as completing eligibility, but omitted Medicaid's priority pathway. Child 1 is Medicaid-eligible in the OLDER_CHILD category, and that eligibility categorically excludes CHIP." +us,scenario_008,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used the shortcut that a child in a low-income household qualifies for CHIP and failed to distinguish Medicaid from CHIP. Child 1 instead qualifies for Medicaid as an OLDER_CHILD, which makes the CHIP result 0." +us,scenario_008,child1_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated being under 19 and below New Jersey's CHIP income ceiling as sufficient, omitting the Medicaid-ineligibility prerequisite. Child 1 is Medicaid-eligible in the OLDER_CHILD category and therefore cannot qualify for CHIP." +us,scenario_008,child1_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility directly from age 17 and low household income without applying Medicaid precedence. Child 1 qualifies for Medicaid as an OLDER_CHILD, so CHIP eligibility is precluded." +us,scenario_008,child1_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model compared household MAGI with the CHIP income limit but skipped the rule that CHIP covers children who do not qualify for Medicaid. Child 1 qualifies for Medicaid in the OLDER_CHILD category, yielding CHIP ineligibility." +us,scenario_008,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicit Medicaid-enrollment or eligibility flag as disqualifying instead of deriving eligibility from age, dependent status, state, and household income. Child 1 belongs to New Jersey's OLDER_CHILD category, and the household's MAGI of 0.55 times FPL satisfies its income threshold." +us,scenario_008,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 17 as within the WIC child age range. WIC child eligibility ends at age five, so Child 1 is categorically ineligible regardless of household income." +us,scenario_008,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly treated a 17-year-old as categorically eligible for WIC and also asserted that the household passed the applicable income test. Child 1 is outside WIC's under-five child category, and the traced annual income exceeds the income limit applied by the engine." +us,scenario_008,child2_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated income below New Jersey's CHIP ceiling as sufficient for eligibility. It failed to determine that Child 2 is Medicaid-eligible under the OLDER_CHILD category, which automatically excludes the child from CHIP." +us,scenario_008,child2_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model compared household income only with New Jersey's CHIP upper-income threshold and skipped the prior Medicaid-eligibility screen. Child 2 qualifies for Medicaid under the OLDER_CHILD category and therefore is not eligible for CHIP. +us,scenario_008,child2_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model applied the under-19 age rule, CHIP income ceiling, and lack of listed private coverage but omitted CHIP's requirement that the child not qualify for Medicaid. Child 2 is Medicaid-eligible under the OLDER_CHILD category, so the CHIP result is 0." +us,scenario_008,child2_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model equated being a low-income child with CHIP eligibility without screening for Medicaid first. Child 2 qualifies for Medicaid under the OLDER_CHILD category, and that existing eligibility disqualifies the child from CHIP." +us,scenario_008,child2_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model treated age under 19 and income below the CHIP limit as sufficient conditions. It omitted the Medicaid exclusion: Child 2 is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP. +us,scenario_008,child2_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model used the child's age and low household income as a direct CHIP pathway but did not apply the Medicaid-eligibility prerequisite. Child 2 qualifies for Medicaid under the OLDER_CHILD category, making CHIP eligibility false." +us,scenario_008,child2_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model relied solely on the child's age and household MAGI being within New Jersey's CHIP limits. It failed to screen out Child 2 after the OLDER_CHILD Medicaid pathway established Medicaid eligibility, which requires a CHIP value of 0." +us,scenario_008,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly stated Medicaid trigger as ineligibility instead of deriving the child's mandatory older-child category from age 14 and dependent status. It then failed to apply that category's MAGI income test, which the household passes at 0.55 times FPL." +us,scenario_008,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a WIC child age range of 1–18 and therefore treated age 14 as eligible, although WIC covers children only through age four. It also reversed the traced income comparison by calling $30,915 at or below the applicable limit and treated housing assistance as evidence of eligibility rather than applying the WIC income test." +us,scenario_008,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model incorrectly placed a 14-year-old within the WIC child age category; WIC child eligibility ends at age five. It also skipped the household income test that the computation trace applies. +us,scenario_008,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly treated age 14 as WIC-eligible even though the child category is restricted to children under five. It also asserted that household income was within the WIC limit, contrary to the traced comparison showing approximately $30,915 above the applicable threshold." +us,scenario_008,child3_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated low income and age as sufficient for CHIP and failed to test Medicaid eligibility first. Child3 qualifies for Medicaid under New Jersey's OLDER_CHILD category, which precludes CHIP eligibility; housing-assistance receipt does not establish CHIP eligibility." +us,scenario_008,child3_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied New Jersey CHIP's 355% FPL ceiling as a standalone eligibility test. It omitted the mutually exclusive Medicaid screen: child3's much lower household income qualifies the child for Medicaid under the OLDER_CHILD category, so CHIP eligibility is false." +us,scenario_008,child3_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model concluded that being below New Jersey's CHIP upper-income limit establishes CHIP eligibility. It failed to apply the prior Medicaid eligibility pathway, under which child3 qualifies as an OLDER_CHILD and is therefore excluded from CHIP." +us,scenario_008,child3_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model compared income only with CHIP's 355% FPL ceiling and never evaluated whether child3 qualified for Medicaid. The approximately $30,915 household income places this 12-year-old in New Jersey's OLDER_CHILD Medicaid category, and Medicaid eligibility precludes CHIP." +us,scenario_008,child3_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated age, lack of listed coverage, and income below CHIP's ceiling as sufficient conditions. It omitted the controlling exclusion that a Medicaid-eligible child cannot also be CHIP-eligible; child3 qualifies for Medicaid under the OLDER_CHILD category." +us,scenario_008,child3_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used the shortcut that a low-income child is CHIP-eligible without distinguishing Medicaid from CHIP. The correct eligibility sequence assigns child3 to Medicaid's OLDER_CHILD category, which makes CHIP eligibility false." +us,scenario_008,child3_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model equated low household income with CHIP eligibility and skipped New Jersey's Medicaid determination. Child3 qualifies for Medicaid as an OLDER_CHILD, and that eligibility bars CHIP." +us,scenario_008,child3_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model applied CHIP's 355% FPL ceiling without first screening child3 for Medicaid, even though the stated income qualifies the child under the OLDER_CHILD Medicaid category and therefore precludes CHIP. It also incorrectly added an invented American Opportunity Credit amount to MAGI; a tax credit is not income and no tuition or scholarship amount was listed." +us,scenario_008,child3_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated income within New Jersey's CHIP limit as dispositive and omitted CHIP's Medicaid-ineligibility requirement. At the stated household income, child3 qualifies for Medicaid under the OLDER_CHILD category and consequently is not CHIP-eligible." +us,scenario_008,child3_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separately stated Medicaid trigger instead of deriving eligibility from the supplied age, dependent status, state, household composition, and income. Child 3 belongs to New Jersey's OLDER_CHILD category, and MAGI of 0.55 times FPL satisfies that category's income test." +us,scenario_008,child3_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 12 as within WIC’s child age range. WIC child eligibility ends at age five, and it also failed to apply the household income test that exceeds the 185% federal-poverty limit." +us,scenario_008,child3_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model misapplied both WIC conditions: a 12-year-old is outside the eligible child category, and the household’s approximately $30,915 income exceeds the applicable 185% federal-poverty threshold. Its conclusion therefore rests on an incorrect age rule and an incorrect income-limit comparison." +us,scenario_008,child4_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model treated income below New Jersey's CHIP ceiling as sufficient for CHIP eligibility. It failed to apply the Medicaid-exclusion step: child4 qualifies for Medicaid under the OLDER_CHILD category at this household income and therefore cannot qualify for CHIP. +us,scenario_008,child4_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model checked age and the upper CHIP income limit but omitted the requirement that a CHIP child not qualify for Medicaid. Child4's low household income establishes OLDER_CHILD Medicaid eligibility, which makes CHIP eligibility false." +us,scenario_008,child4_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly concluded that satisfying NJ FamilyCare's age, residency, and broad CHIP income ceiling establishes CHIP eligibility. It never tested the mutually exclusive Medicaid pathway, under which this 10-year-old qualifies as an OLDER_CHILD and is consequently ineligible for CHIP." +us,scenario_008,child4_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model used the 355% FPL CHIP ceiling as a standalone eligibility test. At this much lower income, child4 first qualifies for Medicaid in the OLDER_CHILD category, and Medicaid eligibility bars CHIP eligibility." +us,scenario_008,child4_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model converted the generic facts that child4 is a child in a low-income household directly into CHIP eligibility. The low income instead places child4 in New Jersey's OLDER_CHILD Medicaid pathway, and CHIP excludes children who qualify for Medicaid." +us,scenario_008,child4_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model treated age 10 and low household income as sufficient for New Jersey CHIP. It omitted the prior Medicaid eligibility test: child4 qualifies for Medicaid as an OLDER_CHILD, so CHIP eligibility is false." +us,scenario_008,child4_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model compared MAGI only with New Jersey's 355% FPL CHIP ceiling and treated the absence of listed health coverage as dispositive. It failed to recognize that the same low income qualifies child4 for Medicaid under the OLDER_CHILD category, which independently excludes the child from CHIP." +us,scenario_008,child4_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility solely because child4 is age 10 and household MAGI falls within the broad CHIP income range. It omitted the mutually exclusive OLDER_CHILD Medicaid pathway, for which child4 qualifies and which makes CHIP eligibility false." +us,scenario_008,child4_head_start_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model explicitly treated age 10 as Head Start eligible, misapplying the program’s preschool-age requirement. Child 4 is outside the eligible age range, and the household also fails the applicable income threshold." +us,scenario_008,child4_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required an explicit Medicaid-triggering input instead of deriving eligibility from the supplied age, state, household composition, and income. Child4 is in New Jersey’s OLDER_CHILD category, and MAGI of 0.55 times FPL is below that category’s threshold, so the model should have returned Yes." +us,scenario_008,child4_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 10 as within the WIC child age range. WIC child eligibility ends before age five, and satisfying an age category would still require passing the income test." +us,scenario_008,child4_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model invented a WIC pathway for a school-age child. WIC covers qualifying infants and children under five, not 10-year-olds, and the household also exceeds the income ceiling used in the trace." +us,scenario_008,child4_wic_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly stated that the household was within WIC income limits; the trace counts approximately $30,915 and places it above the applicable ceiling. It also failed to apply WIC's under-five child age restriction to the 10-year-old." +us,scenario_008,child5_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the CHIP age and income tests but omitted the requirement that a CHIP applicant be ineligible for Medicaid. Child 5 qualifies for Medicaid under the YOUNG_CHILD category, so the model's further claim that housing assistance reduces countable income cannot establish CHIP eligibility." +us,scenario_008,child5_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that the 5-year-old was not eligible through a Medicaid-only age-and-income pathway. At this household income, Child 5 qualifies for Medicaid under the YOUNG_CHILD category, which disqualifies the child from CHIP regardless of being below the CHIP ceiling." +us,scenario_008,child5_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated being under age 19 and below New Jersey's CHIP income ceiling as sufficient. It failed to test Medicaid first: Child 5 qualifies under the YOUNG_CHILD Medicaid category and therefore cannot qualify for CHIP. +us,scenario_008,child5_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model compared income only with the roughly 355% FPL CHIP ceiling and treated lack of other health coverage as dispositive. Child 5's low income instead places the child in the YOUNG_CHILD Medicaid category, and Medicaid eligibility bars CHIP eligibility." +us,scenario_008,child5_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated combined NJ FamilyCare coverage with the distinct CHIP eligibility output, stating that qualification for “CHIP/Medicaid” meant CHIP eligibility. PolicyEngine classifies Child 5 as Medicaid-eligible under YOUNG_CHILD, which makes the separate CHIP indicator false." +us,scenario_008,child5_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used low income and child status as a shortcut to CHIP eligibility without applying Medicaid precedence. The same low income makes Child 5 Medicaid-eligible under the YOUNG_CHILD category, so CHIP eligibility is false." +us,scenario_008,child5_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model assumed that a low-income New Jersey child qualifies for CHIP without checking Medicaid eligibility. Child 5 qualifies for Medicaid under the YOUNG_CHILD category, and that eligibility excludes the child from CHIP." +us,scenario_008,child5_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model compared its MAGI estimate with the 355% FPL CHIP ceiling and treated the absence of health coverage as sufficient. It omitted the Medicaid-exclusion step: at this income, Child 5 qualifies for Medicaid under YOUNG_CHILD and is therefore ineligible for CHIP." +us,scenario_008,child5_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated household MAGI within the CHIP income limit as sufficient for eligibility. Child 5 first qualifies for Medicaid under the YOUNG_CHILD category, and Medicaid eligibility automatically prevents CHIP eligibility." +us,scenario_008,child5_early_head_start_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model incorrectly treated Early Head Start as covering children from birth through age 5. Child 5 is age 5, while Early Head Start requires the child to be under age 3, so housing assistance and low household income do not make this child eligible." +us,scenario_008,child5_head_start_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model incorrectly restricted Head Start to ages three and four and treated age five as categorically ineligible. Under the benchmark's Head Start age rule, this five-year-old meets the age condition and also qualifies through SNAP participation and household income below 130% of the applicable poverty guideline." +us,scenario_008,child5_head_start_eligible,deepseek-v4-pro,llm_error,age_disability,False,The model applied a shortcut that excludes every five-year-old as past preschool age. This child meets the Head Start age condition and satisfies both SNAP categorical eligibility and the direct income threshold. +us,scenario_008,child5_head_start_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,The model treated age five as aging the child out of Head Start instead of applying the benchmark's actual age condition. The child remains age-eligible and qualifies through SNAP as well as household income below 130% of the poverty guideline. +us,scenario_008,child5_head_start_eligible,qwen3.8-max,llm_error,age_disability,False,The model incorrectly made age five an automatic Head Start exclusion. The child meets the applicable age condition and the eligibility pathways established by SNAP participation and income below the program threshold. +us,scenario_008,child5_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicit Medicaid status or special eligibility fact as disqualifying and failed to apply the age-based YOUNG_CHILD pathway. Child5’s age of 5 places the dependent in that category, and the computed MAGI of 0.55 times FPL satisfies its income limit." +us,scenario_008,child5_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated “up to age 5” as including a child whose stated age is 5, but WIC child eligibility ends when the child reaches age five. It also incorrectly treated housing assistance as proof of WIC eligibility; housing assistance neither changes the age cutoff nor establishes WIC adjunctive eligibility." +us,scenario_008,child5_wic_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model included completed age 5 in the WIC child category instead of applying the under-five cutoff. It also invented SNAP receipt from a prompt that only requested a SNAP output, so its claimed adjunctive-eligibility pathway was absent from the household facts." +us,scenario_008,child5_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model contradicted the rule it stated: eligibility through the fifth birthday excludes a child whose annual age is already 5. It also treated housing assistance as adjunctive WIC eligibility, although that benefit does not override the categorical age cutoff." +us,scenario_008,child5_wic_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The answer treated age 5 as categorically eligible without applying WIC’s requirement that a participating child be under age five. The correct age test returns false for Child 5. +us,scenario_008,child5_wic_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model interpreted “eligibility ends at age 5” as meaning age 5 qualifies. Reaching age five ends WIC child-category eligibility, so the income comparison cannot make Child 5 eligible." +us,scenario_008,child5_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model placed a five-year-old inside the WIC child age range. PolicyEngine applies an under-five categorical test, which Child 5 fails." +us,scenario_008,child5_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed that the benchmark’s WIC category includes completed age 5. The applicable categorical condition is age below 5, producing no eligibility for Child 5." +us,scenario_008,child5_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,The model declared eligibility from age 5 alone and reversed the WIC age cutoff. A child aged 5 is outside the under-five child category. +us,scenario_008,child5_wic_eligible,inkling,llm_error,categorical_eligibility,False,"The model conflated “through the fifth birthday” with eligibility throughout completed age 5. Child 5 has reached the age cutoff, so its income calculation does not establish WIC eligibility." +us,scenario_008,child5_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated age 5 as categorically eligible and proceeded to an income comparison. WIC’s child category requires age under 5, so Child 5 fails before the asserted household-income threshold can confer eligibility." +us,scenario_008,child5_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated age 5 as satisfying WIC’s child-category rule. Because eligibility ends upon reaching age five, income within the program limit does not make Child 5 eligible." +us,scenario_008,child6_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model compared household income only with New Jersey's upper CHIP limit and treated falling below that limit as sufficient. It omitted the prior Medicaid determination: Child 6 qualifies under the YOUNG_CHILD category and is therefore excluded from CHIP. +us,scenario_008,child6_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied the CHIP age and maximum-income tests without checking whether the child already qualifies for Medicaid. Child 6's age places the child in New Jersey's YOUNG_CHILD Medicaid category, which makes CHIP eligibility false." +us,scenario_008,child6_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated income below the infant CHIP ceiling as establishing CHIP eligibility. It skipped the mutually exclusive Medicaid pathway: the age-1 child is Medicaid-eligible under YOUNG_CHILD and thus cannot qualify for CHIP. +us,scenario_008,child6_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly declared that age, income, and lack of other health coverage exhausted all CHIP conditions. It omitted Medicaid ineligibility as a required condition; Child 6 qualifies for Medicaid under YOUNG_CHILD, which blocks CHIP." +us,scenario_008,child6_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model collapsed Medicaid and CHIP into a single NJ FamilyCare income test and inferred CHIP eligibility from low income. That low income and the child's age instead establish the YOUNG_CHILD Medicaid pathway, making the child ineligible for CHIP." +us,scenario_008,child6_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used low income and child status as a shortcut to CHIP eligibility without evaluating Medicaid first. The age-1 child qualifies for Medicaid under New Jersey's YOUNG_CHILD category, so CHIP eligibility is false." +us,scenario_008,child6_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model compared income with the CHIP upper limit but failed to apply CHIP's Medicaid-ineligibility condition. Child 6 is Medicaid-eligible through the YOUNG_CHILD category and is consequently excluded from CHIP. +us,scenario_008,child6_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model equated being a low-income New Jersey child with CHIP eligibility. At age 1, Child 6 instead qualifies for Medicaid under YOUNG_CHILD, and that Medicaid eligibility precludes CHIP." +us,scenario_008,child6_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model relied on housing assistance, low income, and no listed health coverage while omitting the required Medicaid screen. Child 6 qualifies for Medicaid under the YOUNG_CHILD category, so the child is not eligible for CHIP." +us,scenario_008,child6_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated MAGI within New Jersey's CHIP income limit as sufficient for eligibility. It failed to determine first that the age-1 child is Medicaid-eligible under YOUNG_CHILD, which makes CHIP eligibility false." +us,scenario_008,child6_early_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model reversed the Early Head Start age rule by treating age 1 as disqualifying. A one-year-old is within the program's under-3 age group, and the household also passes the income test, so the correct binary output is 1." +us,scenario_008,child6_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 1 as within the preschool-age Head Start range. Child 6 belongs to the Early Head Start age category, not Head Start for preschool-age children, and the household's income also exceeds the applicable Head Start cutoff." +us,scenario_008,child6_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separately stated Medicaid eligibility trigger instead of applying the age-based YOUNG_CHILD pathway. Child 6 is age 1 and the household's MAGI is 0.55 times FPL, so the young-child income test yields Medicaid eligibility." us,scenario_008,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_008,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model incorrectly put the household into the EITC phaseout and reduced the $8,231 credit to $8,173. It also stated the refundable CTC calculation was about $4,199 but submitted a total inconsistent with even its own component amounts instead of $8,231 + $4,202.61." -us,scenario_008,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model reduced the EITC to $1,395 instead of applying the $8,231 three-or-more-child amount, then replaced its own $4,245 refundable-CTC calculation with an unexplained $1,305. It also incorrectly counted the age-17 child as CTC-qualifying." -us,scenario_008,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's stated components total only about $12,469, but it submitted $13,167 after an unsupported $698 adjustment. The traced components are $8,231 of EITC and $4,202.61 of refundable CTC." -us,scenario_008,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model granted a $1,000 refundable American Opportunity Credit despite the instruction that unlisted qualified education expenses equal zero. It then reduced refundable CTC to $388 rather than applying the $4,202.61 earned-income-limited amount." -us,scenario_008,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model understated EITC at roughly $7,700 instead of $8,231 and added a refundable AOTC despite zero listed qualified education expenses. Refundable CTC is $4,202.61, so the two applicable components total $12,433.61." -us,scenario_008,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used 2025 EITC parameters, including a $26,511 phaseout threshold and $8,046 maximum, rather than the applicable 2026 parameters. That wrongly reduced EITC to $7,178 instead of $8,231; the refundable CTC component is $4,202.61." -us,scenario_008,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated the $1,700-per-child refundable limit as producing $8,500 and ignored that the aggregate refundable CTC is limited by 15% of earned income above the threshold, which yields $4,202.61. It also added a refundable AOTC with zero qualified education expenses and understated the EITC." -us,scenario_008,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an incorrect $3,000 refundable-CTC earnings threshold and an incorrect EITC amount of $7,787. The applicable components are an $8,231 EITC and $4,202.61 refundable CTC." -us,scenario_008,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly reduced the three-or-more-child EITC to $7,691. The EITC is $8,231, and combining it with the $4,202.61 refundable CTC yields $12,433.61." -us,scenario_008,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied no component calculation and its $11,400 total does not equal the applicable $8,231 EITC plus $4,202.61 refundable CTC. Its claim that the household receives the full Child Tax Credit also fails to apply the refundable CTC earned-income limit." -us,scenario_008,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model approximated the EITC as $8,200 and understated refundable CTC as $4,124. The traced amounts are $8,231 and $4,202.61, respectively." -us,scenario_008,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used incorrect values for both components, reducing EITC to $7,128 and refundable CTC to $4,128. The 2026 calculations produce $8,231 and $4,202.61." -us,scenario_008,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used the obsolete $8,046 EITC maximum instead of the 2026 $8,231 amount. It also treated half the self-employment-tax deduction as a reduction to earned income; the engine's refundable CTC is $4,202.61." -us,scenario_008,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for federal_refundable_credits. -us,scenario_008,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model ignored the six explicitly listed children and the household's wages and self-employment income, falsely concluding that no qualifying-child or earned-income structure was established. Those facts generate an $8,231 EITC and $4,202.61 refundable CTC." -us,scenario_008,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model disregarded the specified wages, self-employment income, joint household, and six children and therefore denied both refundable credits. The household qualifies for $8,231 of EITC and $4,202.61 of refundable CTC." -us,scenario_008,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model added the full $4,000 of self-employment income to wages when calculating the refundable CTC, producing $4,245. The engine applies the relevant net self-employment earnings treatment, yielding $4,202.61; the $8,231 EITC was correctly identified." -us,scenario_008,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used an $8,046 EITC instead of $8,231 and calculated refundable CTC from gross self-employment income, producing $4,245 instead of $4,202.61." -us,scenario_008,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model correctly identified the $8,231 EITC but used gross self-employment income in the refundable-CTC earnings base, producing $4,245. Net earnings from self-employment make refundable CTC $4,202.61." -us,scenario_008,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its submitted total implies the same $8,231 EITC plus a $4,245 refundable CTC obtained by adding gross self-employment income to wages. The refundable CTC earnings base uses net self-employment earnings and yields $4,202.61." -us,scenario_008,federal_refundable_credits,grok-4.3,llm_error,other,False,"The model confused refundable credits with nonrefundable credits and treated zero pre-refund income tax as eliminating EITC and refundable CTC. Refundable credits remain payable beyond tax liability, producing $8,231 of EITC and $4,202.61 of refundable CTC." -us,scenario_008,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an incorrect $3,000 refundable-CTC earnings threshold and pre-TCJA child-credit parameters, yielding $4,124. It also overstated EITC at $8,247 rather than applying the $8,231 amount." -us,scenario_008,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly placed the household in the EITC phaseout and reduced EITC to $7,901. The applicable EITC is $8,231, and refundable CTC is $4,202.61." -us,scenario_008,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for federal_refundable_credits. -us,scenario_008,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly used the $8,231 EITC but calculated refundable CTC from wages plus gross self-employment income, obtaining $4,245. Applying net self-employment earnings produces $4,202.61." -us,scenario_008,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used an incorrect EITC maximum of about $7,152 and an incorrect $28,120 phaseout threshold, reducing EITC to $6,588. It also used gross self-employment income to calculate $4,245 of refundable CTC instead of $4,202.61." -us,scenario_008,federal_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model awarded a $1,000 refundable American Opportunity Credit even though qualified education expenses are unlisted and therefore zero. It also used outdated estimated EITC parameters to reduce EITC to $7,144 and counted six children for CTC even though the age-17 child is not CTC-qualifying; the actual applicable components are $8,231 and $4,202.61." -us,scenario_008,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separately provided school-meals eligibility or direct-certification indicator. It failed to derive SNAP categorical eligibility and independently failed to apply the free-meal income test, under which household income is 55% of the federal poverty guideline and therefore below the 130% threshold." -us,scenario_008,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model applied the New Jersey Medicaid income guideline incorrectly. The head's MAGI is 0.55 of FPL, below the ACA adult expansion limit, so the income test is passed rather than exceeded." -us,scenario_008,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring an explicit benchmark-provided eligibility indicator instead of computing the ACA adult expansion category from the facts. The head is a non-dependent, non-elderly adult in New Jersey with MAGI at 0.55 of FPL, which is the qualifying MAGI-based adult pathway." -us,scenario_008,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model required disability for adult Medicaid eligibility and skipped the ACA adult expansion category. In New Jersey, a non-disabled adult under age 65 qualifies through the MAGI adult expansion pathway when income is below the expansion threshold, and this head's MAGI is 0.55 of FPL." -us,scenario_008,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly identified the ACA adult expansion income pathway but then invented a Medicaid disqualification for receiving housing assistance. Housing assistance does not make adults ineligible for Medicaid under this PolicyEngine rule, so the head remains eligible with MAGI at 0.55 of FPL." -us,scenario_008,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model converted WIC's individual categorical test into household categorical eligibility by treating the infant and age-5 child as making the head eligible. The head is not pregnant, postpartum, breastfeeding, an infant, or a child under age 5, so the qualifying child in the household does not confer WIC eligibility on the head." +us,scenario_008,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model incorrectly phased down the EITC from $8,231 at this income instead of awarding the full $8,231, and its submitted $12,130 does not follow from either of its own stated component totals. The correct components are $8,231 of EITC and $4,202.61 of refundable CTC." +us,scenario_008,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model reduced the EITC to $1,395 instead of applying the 2026 three-or-more-child amount of $8,231, then replaced its correctly estimated $4,245 CTC phase-in with an unexplained $1,305. It also counted the age-17 child as under 17." +us,scenario_008,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The submitted $13,167 contradicts the model's own stated calculation of approximately $8,267 plus $4,202, which totals about $12,469. Using the traced components gives $8,231 plus $4,202.61, or $12,433.61." +us,scenario_008,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented a $1,000 refundable AOTC even though the prompt sets unlisted qualified education expenses to zero, and then reduced the refundable CTC to $388 despite correctly deriving its roughly $4,200 earned-income limit. The refundable CTC is $4,202.61 and no refundable AOTC applies." +us,scenario_008,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model understated the EITC at roughly $7,700 instead of $8,231 and added a refundable AOTC despite zero listed qualified education expenses. Only the $8,231 EITC and $4,202.61 refundable CTC enter the total." +us,scenario_008,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used 2025 EITC parameters, including a $26,511 phaseout threshold and $8,046 maximum, instead of the applicable 2026 parameters. That obsolete phaseout produced $7,178 rather than the traced $8,231 EITC." +us,scenario_008,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated the $1,700-per-child refundable CTC ceiling as automatically producing $8,500 and ignored the lower household-level limit of 15% of earned income above $2,500. It also assumed a $1,000 AOTC despite zero education expenses; the refundable CTC is $4,202.61 and the AOTC is zero." +us,scenario_008,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an incorrect $3,000 refundable-CTC earnings threshold and reduced the EITC to $7,787 under incorrect 2026 parameters. The calculation uses the applicable earned-income rule to produce $4,202.61 of refundable CTC and $8,231 of EITC." +us,scenario_008,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly placed refundable CTC near $4,199 but incorrectly phased the EITC down to $7,691. The 2026 EITC contribution is $8,231, and the precise refundable CTC is $4,202.61." +us,scenario_008,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The unexplained $11,400 total does not implement the two applicable components. The full traced amounts are $8,231 of EITC and $4,202.61 of refundable CTC, totaling $12,433.61." +us,scenario_008,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model approximated both components rather than applying their exact 2026 computations, understating the refundable CTC at $4,124 and the EITC at about $8,200. The components are $4,202.61 and $8,231." +us,scenario_008,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied an erroneous EITC phaseout that reduced the credit to $7,128 and understated the refundable CTC at $4,128. The applicable results are $8,231 and $4,202.61." +us,scenario_008,federal_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used half of self-employment tax to reduce earned income, producing $30,517, and then used stale $8,046 EITC parameters. Half the self-employment tax is an AGI deduction, while the credit computations yield $8,231 of EITC and $4,202.61 of refundable CTC." +us,scenario_008,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation, so it failed the required output contract." +us,scenario_008,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model ignored the six explicitly listed children and the household's earned income, falsely concluding that no qualifying-child EITC or refundable CTC claim was established. Those facts produce $8,231 of EITC and $4,202.61 of refundable CTC." +us,scenario_008,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed that no qualifying income structure was specified even though the prompt lists $26,800 of wages, $4,000 of self-employment income, a joint couple, and six children. Those facts support the two refundable credits totaling $12,433.61." +us,scenario_008,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used all $4,000 of gross self-employment income in earned income, calculating refundable CTC as $4,245. The applicable earned-income computation produces refundable CTC of $4,202.61; adding the correctly identified $8,231 EITC gives $12,433.61." +us,scenario_008,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a stale $8,046 EITC amount and gross self-employment income to obtain a $4,245 refundable CTC. The 2026 components are $8,231 and $4,202.61." +us,scenario_008,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model correctly identified the $8,231 EITC but treated gross self-employment income as earned income when calculating the refundable CTC, yielding $4,245. The refundable CTC is $4,202.61." +us,scenario_008,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $12,476 answer implies the same $8,231 EITC plus a $4,245 refundable CTC obtained by adding gross self-employment income to wages. The applicable refundable-CTC computation yields $4,202.61." +us,scenario_008,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model treated refundable credits as unavailable once nonrefundable credits reduced income tax to zero. EITC and refundable CTC are payable beyond tax liability, producing $8,231 and $4,202.61 respectively." +us,scenario_008,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an obsolete $3,000 refundable-CTC threshold and pre-TCJA per-child rules, while also overstating the EITC at $8,247. The applicable 2026 amounts are $8,231 of EITC and $4,202.61 of refundable CTC." +us,scenario_008,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly phased the EITC down to $7,901 instead of awarding $8,231. Its refundable CTC estimate was close, but the precise amount is $4,202.61." +us,scenario_008,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model overstated the EITC maximum at $8,271 and calculated refundable CTC from $30,800 of gross wages plus self-employment income, producing $4,245. The traced amounts are $8,231 and $4,202.61." +us,scenario_008,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation, so it failed the required output contract." +us,scenario_008,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly used the $8,231 EITC but included gross self-employment income in the refundable-CTC earnings base, giving $4,245. The correct refundable CTC is $4,202.61." +us,scenario_008,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used an incorrect $7,152 EITC maximum and an incorrect $28,120 phaseout threshold, reducing EITC to $6,588. It also used gross self-employment income in the CTC phase-in; the correct components are $8,231 and $4,202.61." +us,scenario_008,federal_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model awarded a $1,000 refundable AOTC despite the instruction that unlisted education expenses equal zero and used stale EITC phaseout parameters to reduce EITC to $7,144. No AOTC applies, and the EITC is $8,231 alongside $4,202.61 of refundable CTC." +us,scenario_008,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model wrongly limited refundable CTC by remaining income-tax liability, even though the refundable portion is payable beyond liability and equals $4,202.61 here. It also understated EITC at $6,801, and its stated components do not arithmetically equal its submitted $6,586." +us,scenario_008,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a school-meals eligibility indicator in the prompt instead of deriving eligibility from program and income rules. PolicyEngine finds all household members SNAP-eligible, establishing categorical eligibility, and the household's 55% federal-poverty-guideline ratio also independently satisfies the 130% free-meal limit." +us,scenario_008,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly placed the head above New Jersey's Medicaid income limit. PolicyEngine computes the relevant MAGI level as 0.55 FPL, well below the ACA adult expansion threshold." +us,scenario_008,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model required an explicit eligibility indicator instead of deriving eligibility from the supplied household facts. The 42-year-old nondependent head qualifies through New Jersey's ACA adult expansion category because MAGI is 0.55 FPL. +us,scenario_008,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model treated disability as necessary for an adult to receive Medicaid. New Jersey's ACA expansion provides a separate MAGI-based pathway for nondisabled adults under 65, and the head qualifies at 0.55 FPL." +us,scenario_008,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model invented a Medicaid disqualification for adults receiving housing assistance after correctly identifying the ACA expansion pathway and finding income below its threshold. Housing assistance does not bar MAGI-based Medicaid eligibility, so the head remains eligible in the ADULT category." +us,scenario_008,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated the infant's individual WIC categorical eligibility as extending to the Head through household membership. WIC requires the Head personally to fall within a qualifying category, and the listed facts establish none for this 42-year-old adult." us,scenario_008,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_008,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model replaced the applicable New Jersey TDI and FLI contributions of $50.92 and $61.64 with UI, workforce, and estimated FLI charges totaling $128.65. Adding the correct $112.56 state contribution to $2,050.20 of federal FICA yields $2,162.76." -us,scenario_008,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model’s reasoning became arithmetically inconsistent and ultimately reduced New Jersey employee payroll taxes to a fictitious $2.12 assessment. The applicable state charges are $50.92 of TDI and $61.64 of FLI, totaling $112.56." -us,scenario_008,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,The model stopped at federal FICA because it treated the New Jersey contribution rates as uncertain. It omitted $50.92 of employee TDI and $61.64 of employee FLI from the payroll-tax output. -us,scenario_008,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model explicitly set New Jersey mandatory employee payroll taxes to zero. The output includes $50.92 of TDI and $61.64 of FLI in addition to $2,050.20 of federal FICA." -us,scenario_008,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model incorrectly excluded New Jersey employee TDI and FLI from the measure and reported only rounded federal FICA. Those state contributions add $112.56, and federal FICA itself is $2,050.20 rather than $2,050.00." -us,scenario_008,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,The model invoked a “traditional payroll tax” interpretation that contradicts the prompt’s express inclusion of mandatory employee state payroll taxes. It therefore omitted New Jersey TDI of $50.92 and FLI of $61.64. -us,scenario_008,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model acknowledged New Jersey UI/DI/FLI contributions but set them to zero because it did not resolve their rates. The applicable components are TDI of $50.92 and FLI of $61.64, which add $112.56 to federal FICA." -us,scenario_008,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model applied incorrect New Jersey rates of 0.10% for TDI and 0.09% for FLI, producing only $50.92 combined. The computed contributions are $50.92 for TDI and $61.64 for FLI, or $112.56 combined." -us,scenario_008,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model used an approximate 0.515% New Jersey SUI/FLI charge of $138.02 instead of the computed TDI and FLI components. Those components total $112.56, making total payroll tax $2,162.76." -us,scenario_008,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model applied only the 7.65% federal FICA rate and rounded the result to $2,050. It omitted $50.92 of New Jersey TDI and $61.64 of New Jersey FLI." -us,scenario_008,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model estimated New Jersey SUI/FLI/TDI at roughly $138 rather than computing the applicable components. New Jersey TDI is $50.92 and FLI is $61.64, totaling $112.56." -us,scenario_008,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model substituted an estimated $123.80 New Jersey UI/FLI amount for the applicable state contributions. TDI and FLI equal $50.92 and $61.64, respectively, for a combined $112.56." -us,scenario_008,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model overstated New Jersey employee payroll taxes as $138.02. The applicable TDI and FLI charges total $112.56, so they raise $2,050.20 of federal FICA to $2,162.76." -us,scenario_008,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no payroll-tax output or derivation. It therefore failed the required structured-output contract. -us,scenario_008,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly stated federal FICA components totaling $2,050.20 but then submitted $4,097 without a supporting computation. It also omitted the $112.56 of New Jersey TDI and FLI required to reach $2,162.76." -us,scenario_008,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The submitted $1,525 is inconsistent with the standard 6.2% Social Security and 1.45% Medicare rates the model claimed to apply to $26,800. Correct federal FICA is $2,050.20, followed by $112.56 of New Jersey TDI and FLI." -us,scenario_008,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model applied a 0.9425% bundle of New Jersey UI, workforce, TDI, and FLI charges, overstating the state portion as $252.59. This output includes only the computed employee TDI and FLI amounts, totaling $112.56." -us,scenario_008,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model overstated New Jersey employee payroll contributions as $253.26. The applicable TDI and FLI contributions are $50.92 and $61.64, totaling $112.56." -us,scenario_008,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"The model used an unsupported $263.98 estimate for New Jersey employee payroll contributions. The state portion is $112.56 from TDI and FLI, which combines with $2,050.20 of federal FICA." -us,scenario_008,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,The model overstated the New Jersey portion as $202.34. The applicable employee TDI and FLI contributions total $112.56. -us,scenario_008,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model calculated only federal FICA at 7.65% and rounded it to $2,050. It omitted New Jersey employee TDI of $50.92 and FLI of $61.64." -us,scenario_008,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"The model used a combined 0.18% New Jersey TDI/FLI rate, producing only $48.24. The computed TDI and FLI amounts are $50.92 and $61.64, totaling $112.56." -us,scenario_008,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model expressly assumed no state payroll taxes and returned rounded federal FICA alone. New Jersey TDI and FLI add $112.56 to the exact federal amount of $2,050.20." -us,scenario_008,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll-tax output or explanation. It therefore failed the required structured-output contract. -us,scenario_008,payroll_tax,kimi-k3,llm_error,payroll_tax_base,False,"The model included UI and workforce charges and overstated New Jersey employee contributions as $257.95. The payroll-tax computation uses TDI of $50.92 and FLI of $61.64, totaling $112.56." -us,scenario_008,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model stopped at federal Social Security and Medicare taxes and rounded their $2,050.20 total to $2,050. It omitted $112.56 of New Jersey employee TDI and FLI." -us,scenario_008,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model substituted a $102.51 New Jersey UI/workforce charge and set disability and family-leave contributions to zero. The applicable state components are TDI of $50.92 and FLI of $61.64, totaling $112.56." -us,scenario_008,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated low income and housing assistance as grounds for reduced-price meals without first assigning the household to the free-meals tier. At 55% of the federal poverty guideline and with categorical eligibility, the household qualifies for free meals, which makes the mutually exclusive reduced-price output 0." -us,scenario_008,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model did not submit a parseable value for self_employment_tax. The correct computation applies 92.35% to the $4,000 of self-employment income and then the full 15.3% self-employment tax rate, yielding $565.18." -us,scenario_008,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model used the employee-side 7.65% payroll tax rate on the $4,000 of self-employment income, producing $306, while describing it as the 15.3% self-employment rate. Self-employment tax applies the full 15.3% rate to net self-employment earnings after the 92.35% adjustment, so the tax is $4,000 × 0.9235 × 0.153 = $565.18." -us,scenario_008,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not submit a parseable value for self_employment_tax. The correct computation applies 92.35% to the $4,000 of self-employment income and then the full 15.3% self-employment tax rate, yielding $565.18." -us,scenario_008,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model used an incorrect $258 standard deduction instead of $299 and never produced a coherent maximum-allotment calculation: its own formula yields $14,568, but it replaced that result with an unsupported $687 monthly benefit. The correct monthly net income is $1,763.92, and the applicable maximum allotments produce $15,246.91 annually." -us,scenario_008,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a drastically understated $1,164 maximum monthly allotment for an eight-person household and omitted the 20% earned-income deduction. The correct computation uses a $513.33 earned-income deduction, a $299 standard deduction, and eight-person maximum allotments of $1,789–$1,829." -us,scenario_008,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an approximate $280 standard deduction and a $1,756 maximum allotment instead of the applicable $299 deduction and $1,789–$1,829 allotments. Those parameter errors understated the annual benefit." -us,scenario_008,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used a $258 standard deduction and a $1,756 maximum allotment, then reduced its own $13,728 calculation to $11,352 through an unsupported adjustment. The correct parameters yield $1,763.92 monthly net income and $15,246.91 annually." -us,scenario_008,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model estimated net income at roughly $1,700 and the benefit at $1,040 per month without applying the traced $299 standard deduction and calendar-year maximum allotments. Correct net income is $1,763.92 per month, with benefits of approximately $1,260–$1,302 across the year." -us,scenario_008,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model's decisive error was using a $1,536 maximum allotment for an eight-person household instead of the applicable $1,789–$1,829 amounts; it also used a $258 rather than $299 standard deduction. These errors reduced its monthly benefit to $995 instead of approximately $1,260–$1,302." -us,scenario_008,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented a large shelter deduction and estimated annual net income near $8,800 even though the listed rent does not exceed half of adjusted income, so the excess-shelter deduction is zero. The traced deductions total $812.33 monthly and leave $1,763.92 of net income." -us,scenario_008,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model correctly reached approximately $1,763 of monthly net income but used an unsupported $1,900 maximum allotment. The applicable eight-person maximum is $1,789–$1,829, producing $15,246.91 rather than $16,453.44." -us,scenario_008,snap,gemini-3-flash-preview,llm_error,period_annualization,False,"The rounded $15,300 estimate replaces the calendar-year computation with a single approximate annual amount. Applying the two maximum-allotment periods to $1,763.92 of monthly net income yields $15,246.91." -us,scenario_008,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies an overstated monthly allotment of about $1,367 without identifying the deductions or maximum-allotment parameters. The correct contribution against the applicable maximum allotments produces approximately $1,260–$1,302 per month and $15,246.91 annually." -us,scenario_008,snap,gemini-3.1-pro-preview,llm_error,period_annualization,False,"The model treated roughly $1,307 as a constant monthly benefit and multiplied it by 12. The calendar year spans maximum-allotment periods yielding approximately $1,260–$1,302 per month, totaling $15,246.91 rather than $15,684." -us,scenario_008,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied an excess-shelter deduction using a utility allowance even though utilities were unlisted and therefore zero under the prompt. With rent below half of adjusted income, the shelter deduction is zero and monthly net income remains $1,763.92." -us,scenario_008,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $287 standard deduction and a $1,756 maximum allotment instead of $299 and the applicable $1,789–$1,829 maximums. Its shelter conclusion was correct, but the wrong parameters understated the annual benefit." -us,scenario_008,snap,glm-5.2,llm_error,thresholds_rates,False,"The model treated the standard deduction as only $198 annually rather than $299 monthly and used a $1,748 maximum allotment. The correct monthly deduction total is $812.33, leaving $1,763.92 of net income for the benefit formula." -us,scenario_008,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no positive benefit was supportable despite the household passing both income tests and qualifying categorically through TANF non-cash assistance. The benefit formula yields approximately $1,260–$1,302 per month." -us,scenario_008,snap,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the supplied facts as inadequate instead of applying the prompt's zero defaults and the standard SNAP deductions. The listed income, household size, and zero-default expenses fully determine $1,763.92 in monthly net income and a positive allotment." -us,scenario_008,snap,gpt-5.5,llm_error,period_annualization,False,"The model rounded the benefit to a constant $1,260 per month and ignored the higher maximum allotment applicable during part of calendar year 2026. The period-specific monthly benefits total $15,246.91, not $15,120." -us,scenario_008,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's $1,139 monthly estimate understates the applicable eight-person maximum allotments after the income contribution. Using $1,763.92 of monthly net income produces approximately $1,260–$1,302 per month." -us,scenario_008,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The answer reflects a rounded constant monthly estimate rather than the two SNAP parameter periods within calendar year 2026. The applicable monthly amounts sum to $15,246.91, not $15,144." -us,scenario_008,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model cited the earned-income and standard deductions but did not apply the traced parameter values and maximum allotments precisely. The correct $1,763.92 monthly net income and period-specific maximums yield $15,246.91." -us,scenario_008,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly treated housing assistance and income as eliminating SNAP eligibility. The household passes the gross and net income tests, qualifies categorically through TANF non-cash assistance, and receives a positive allotment." -us,scenario_008,snap,grok-4.5,llm_error,thresholds_rates,False,"The model overstated the projected annual maximum allotment at about $21,870 and deducted medical expenses that do not exceed the applicable threshold. With no medical or shelter deduction, the traced net income and applicable maximums produce $15,246.91." -us,scenario_008,snap,grok-build-0.1,llm_error,asset_resource,False,"The model applied the ordinary $4,250 resource limit despite New Jersey broad-based categorical eligibility through TANF non-cash assistance. That categorical pathway removes the asset test, so the $69,500 bank balance does not eliminate SNAP eligibility." -us,scenario_008,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so it failed the required output contract." -us,scenario_008,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly calculated $1,763.92 in monthly net income but used the $1,789 maximum allotment for all twelve months. Calendar year 2026 also includes months governed by the $1,829 maximum, raising the annual total to $15,246.91." -us,scenario_008,snap,minimax-m3,llm_error,asset_resource,False,"The model applied a $3,750 resource limit despite the household's broad-based categorical eligibility through TANF non-cash assistance. New Jersey's categorical pathway removes the asset test, so the bank balance does not make the household ineligible." -us,scenario_008,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a $202 monthly standard deduction instead of $299 and an unsupported $1,838 maximum allotment, then replaced its own $15,391 calculation with an unexplained $14,784 adjustment. The traced parameters yield $1,763.92 monthly net income and $15,246.91 annually." -us,scenario_008,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply the ACA adult expansion pathway for a 41-year-old New Jersey adult and treated Medicaid as requiring additional unspecified benchmark trigger facts. PolicyEngine's relevant facts establish a MAGI category adult with income at 0.55 FPL, which is below the 138% FPL expansion threshold." -us,scenario_008,spouse_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly identified income below the New Jersey adult Medicaid threshold but invented a housing-assistance disqualification. Housing assistance does not disqualify adults from Medicaid under this PolicyEngine eligibility computation, and the spouse qualifies through the MAGI-based ACA adult expansion category." -us,scenario_008,spouse_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the spouse's disability flag as automatic Medicare eligibility regardless of age. The benchmark requires a qualifying Medicare entitlement rather than disability alone, and the 41-year-old spouse has none." -us,scenario_008,spouse_medicare_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that disability by itself qualifies an individual for Medicare regardless of age. At age 41, the spouse does not satisfy the age rule or any listed alternative Medicare-entitlement pathway." -us,scenario_008,spouse_medicare_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model misrepresented PolicyEngine's disability input as a Medicare-qualifying status. The spouse's age is 41, and the facts supply no separate Medicare entitlement that overrides the age threshold." -us,scenario_008,spouse_medicare_eligible,kimi-k3,llm_error,categorical_eligibility,False,"The model applied a nonexistent rule that every person marked disabled and under 65 is Medicare eligible. Disability alone does not establish the required Medicare entitlement, so the spouse remains ineligible at age 41." -us,scenario_008,spouse_medicare_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model inferred SSDI receipt and completion of Medicare's 24-month disability waiting period from disability and housing assistance, despite the instruction not to infer unlisted benefit receipt. Housing assistance does not establish SSDI entitlement, and neither SSDI receipt nor the waiting-period facts were provided." -us,scenario_008,spouse_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"This model converted the presence of a 1-year-old child into categorical eligibility for the spouse as a caretaker. WIC does not make adult caretakers eligible merely because an eligible child lives in the household; the spouse lacks pregnancy, postpartum, or breastfeeding status, so the person-level categorical test fails." -us,scenario_008,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"This model treated the spouse as categorically eligible because she is the mother in a household with young children. WIC eligibility attaches to pregnant, postpartum, or breastfeeding women and to infants or children under 5, not to a parent as a household representative, so the spouse fails the categorical requirement despite household income being below 185% FPL." -us,scenario_008,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"This model invented a breastfeeding or postpartum pathway from the presence of a 1-year-old child and called it categorical eligibility. The prompt makes unlisted boolean/status facts false, and no pregnancy, postpartum, or breastfeeding status is listed for the spouse, so she is not an eligible WIC category." +us,scenario_008,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model substituted New Jersey unemployment, workforce-development, and family-leave estimates for the traced state components. It needed to add TDI of $50.92 and FLI of $61.64 to federal FICA, rather than adding its $128.65 bundle and setting TDI to zero." +us,scenario_008,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model discarded its own New Jersey calculations and reduced the state contribution to an invented $2.12 rounding adjustment. The required state amount is $112.56, comprising $50.92 of TDI and $61.64 of FLI." +us,scenario_008,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,The model stopped at federal FICA because it treated the 2026 New Jersey contribution rates as uncertain. It omitted $50.92 of employee TDI and $61.64 of employee FLI. +us,scenario_008,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model explicitly set mandatory New Jersey employee payroll taxes to zero. The output includes $50.92 of TDI and $61.64 of FLI in addition to $2,050.20 of federal FICA." +us,scenario_008,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model incorrectly excluded New Jersey employee TDI and FLI from the measure and also rounded $2,050.20 of federal FICA to $2,050. It needed to add the two state contributions totaling $112.56." +us,scenario_008,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model treated New Jersey TDI as employer-funded and dismissed FLI as outside the payroll-tax definition. Under this output, employee TDI is $50.92 and employee FLI is $61.64, so both must be added to federal FICA." +us,scenario_008,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,The model acknowledged New Jersey mandatory contributions but then set them to zero because it lacked confidence in its estimated combined rate. The trace applies exactly $50.92 of TDI and $61.64 of FLI. +us,scenario_008,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used incorrect New Jersey rates, producing TDI of $26.80 and FLI of $24.12. The applicable calculations produce $50.92 and $61.64, respectively." +us,scenario_008,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model used an approximate $138.02 bundle of New Jersey SUI and FLI instead of the traced TDI and FLI components. Those components total $112.56, not $138.02." +us,scenario_008,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model calculated only federal Social Security and Medicare tax and omitted New Jersey employee TDI and FLI. It also rounded federal FICA from $2,050.20 to $2,050." +us,scenario_008,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model replaced the applicable New Jersey TDI and FLI calculations with a rough $138 estimate covering SUI, FLI, and TDI. The state portion is exactly $112.56: $50.92 of TDI plus $61.64 of FLI." +us,scenario_008,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,The model estimated $123.80 of New Jersey UI and FLI rather than calculating the included TDI and FLI contributions. The correct state components total $112.56. +us,scenario_008,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model added $138.02 of estimated New Jersey SUI and FLI. The output instead includes $50.92 of TDI and $61.64 of FLI, totaling $112.56." +us,scenario_008,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required output contract." +us,scenario_008,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly stated federal FICA components totaling $2,050.20 but then submitted $4,097, which does not follow from its arithmetic. It also omitted the $112.56 New Jersey TDI and FLI amount." +us,scenario_008,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The submitted $1,525 is inconsistent with applying the stated standard Social Security and Medicare rates to $26,800 of wages, which alone yields $2,050.20. The model also omitted $112.56 of New Jersey TDI and FLI." +us,scenario_008,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model applied a 0.9425% New Jersey bundle containing UI, workforce, TDI, and FLI contributions. The traced output includes only TDI of $50.92 and FLI of $61.64, totaling $112.56 rather than $252.59." +us,scenario_008,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model overestimated New Jersey employee contributions at $253.26. The applicable TDI and FLI contributions are $50.92 and $61.64, totaling $112.56." +us,scenario_008,payroll_tax,gpt-5.6-sol,llm_error,thresholds_rates,False,The model overestimated mandatory New Jersey employee contributions at $263.98. The required state amount is $112.56 from TDI and FLI. +us,scenario_008,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,The model used $202.34 for New Jersey employee payroll contributions. The traced TDI and FLI amounts total only $112.56. +us,scenario_008,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model included only 7.65% federal FICA and omitted $112.56 of New Jersey employee TDI and FLI. It also rounded $2,050.20 of FICA to $2,050." +us,scenario_008,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"The model applied a combined 0.18% rate to New Jersey TDI and FLI, producing only $48.24. The applicable contributions are $50.92 and $61.64, totaling $112.56." +us,scenario_008,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,The model incorrectly asserted that no state payroll taxes apply and stopped at rounded federal FICA. It omitted New Jersey TDI of $50.92 and FLI of $61.64. +us,scenario_008,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model estimated a $127 New Jersey UI-and-FLI addition and rounded the federal components. The included state levies are TDI of $50.92 and FLI of $61.64, totaling $112.56." +us,scenario_008,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required output contract." +us,scenario_008,payroll_tax,kimi-k3,llm_error,payroll_tax_base,False,"The model added $257.95 for New Jersey UI/workforce, TDI, and FLI contributions. The output includes only TDI of $50.92 and FLI of $61.64, totaling $112.56." +us,scenario_008,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model stopped at federal Social Security and Medicare tax and omitted New Jersey employee TDI and FLI totaling $112.56. It also rounded federal FICA from $2,050.20 to $2,050." +us,scenario_008,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,The model included $102.51 of New Jersey UI while setting TDI and FLI to zero based on older-year assumptions. The traced 2026 state portion instead consists of $50.92 of TDI and $61.64 of FLI. +us,scenario_008,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,The model correctly calculated federal FICA but treated it as the entire employee payroll-tax output. It omitted New Jersey employee TDI of $50.92 and FLI of $61.64. +us,scenario_008,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated low income and housing assistance as grounds for reduced-price meals without checking the mutually exclusive free-meals tier. The household's 55% FPG ratio is below the 130% free-meal threshold, and categorical eligibility independently places it in the FREE tier, so reduced-price eligibility is 0." +us,scenario_008,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output, violating the required output contract." +us,scenario_008,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's $306 answer applies only 7.65% to the $4,000 income despite stating a 15.3% rate. The correct computation applies 15.3% to 92.35% of $4,000, yielding $565.18." +us,scenario_008,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output, violating the required output contract." +us,scenario_008,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model correctly calculated $565.18, then incorrectly subtracted a supposed $282.60 deduction from the tax liability. The deductible one-half of self-employment tax reduces adjusted gross income and is not a credit against self-employment tax, so the liability remains $565.18." +us,scenario_008,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model's own calculation produced $14,568, but it discarded that result and submitted $8,244 using an unexplained $687 monthly figure. It also used an understated maximum allotment and failed to use the traced $812.33 deduction total and $1,763.92 net income." +us,scenario_008,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a severely understated $1,164 maximum monthly allotment for an eight-person household and omitted the 20% earned-income deduction. The applicable maximum is approximately $1,789–$1,829, and total deductions are $812.33 per month." +us,scenario_008,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used a standard deduction of only about $280 annually instead of applying the monthly deduction schedule, leaving net income near $2,040 rather than $1,763.92 per month. It also understated the maximum allotment at $1,756." +us,scenario_008,snap,claude-opus-4.8,llm_error,other,False,"The model correctly derived about $13,728 from its stated assumptions, then replaced it with $11,352 based on unspecified 'income/utility estimates.' That unsupported adjustment abandoned its own computation and omitted the exact deduction and calendar-year allotment schedule." +us,scenario_008,snap,claude-opus-5,llm_error,other,False,"The stated figures are arithmetically inconsistent: a $1,802 maximum less 30% of $1,700 equals about $1,292 monthly, not $1,040. The model therefore submitted a number that does not follow from its own inputs and did not apply the traced monthly parameters." +us,scenario_008,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a $1,536 monthly maximum allotment, far below the applicable $1,789–$1,829 calendar-year amounts. Its net-income calculation was also high because it used a $258 rather than the traced $299 standard deduction." +us,scenario_008,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented net income of roughly $8,800 annually after a utility and shelter deduction, even though the trace yields $1,763.92 monthly net income and no such large shelter reduction. Its $1,691 maximum allotment was also understated." +us,scenario_008,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model closely matched the traced net income but used an unsupported $1,900 monthly maximum allotment. Applying the actual $1,789–$1,829 schedule instead yields the lower annual benefit." +us,scenario_008,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model rounded an exact monthly SNAP calculation to $15,300 without identifying or applying the two calendar-year maximum-allotment parameter periods. The traced monthly deductions, expected contribution, and maximum allotments yield $15,246.91." +us,scenario_008,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"The answer used only household size and a qualitative claim of substantial eligibility, without computing deductions, net income, the 30% contribution, or the maximum allotment. Its $16,400 therefore does not implement the SNAP benefit formula." +us,scenario_008,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model treated approximately $1,307 as the monthly benefit throughout the year. The correct monthly amounts vary around $1,260–$1,302 as calendar-year maximum allotments change, producing $15,246.91 rather than a flat annualization." +us,scenario_008,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied an excess-shelter deduction based on rent and a standard utility allowance even though the traced allowable deductions total $812.33 and net income is $1,763.92. That extra deduction understated the household contribution and overstated SNAP." +us,scenario_008,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $287 standard deduction instead of the traced $299 amount and a flat $1,756 maximum allotment instead of the applicable $1,789–$1,829 schedule. Both parameter errors lowered the annual benefit." +us,scenario_008,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated a $198 standard deduction as annual rather than monthly, leaving annual net income at $24,557 instead of monthly net income of $1,763.92. It also used an understated $1,748 monthly maximum allotment." +us,scenario_008,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model set SNAP to zero without running the eligibility and benefit formula. The household passes the gross and net income tests, qualifies categorically through TANF non-cash assistance, and receives a positive allotment of $15,246.91." +us,scenario_008,snap,gpt-5.4-nano,llm_error,missing_output,False,"The model incorrectly treated the supplied facts as inadequate and defaulted the benefit to zero. Household size, income, housing status, and the prompt's zero-default rules provide the inputs needed to establish eligibility and calculate $15,246.91." +us,scenario_008,snap,gpt-5.5,llm_error,period_annualization,False,"The model flattened the benefit to an estimated $1,260 for all twelve months. Calendar 2026 spans changing maximum-allotment values, with monthly benefits around $1,260–$1,302, so the exact annual total is $15,246.91." +us,scenario_008,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The submitted $1,139 monthly estimate implies either excessive net income or an understated maximum allotment. The traced deductions produce $1,763.92 net income and a benefit of approximately $1,260–$1,302 per month." +us,scenario_008,snap,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model gave no parameter-level computation and understated the exact annual result by using an approximate allotment. Applying the traced $1,763.92 net income and the calendar-year maximum-allotment schedule yields $15,246.91." +us,scenario_008,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,The model cited the earned-income and standard deductions but did not apply the exact monthly maximum-allotment schedule and 30% contribution. Its rounded estimate exceeds the traced annual total by $161.29. +us,scenario_008,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly treated housing assistance and income as eliminating SNAP. The household passes both income tests and categorical eligibility, while housing assistance does not make its SNAP allotment zero." +us,scenario_008,snap,grok-4.5,llm_error,thresholds_rates,False,"The model projected an annual maximum allotment of about $21,870 rather than applying the actual monthly 2026 schedule. It also described an excess medical deduction from $300 of annual OTC expenses even though those expenses do not exceed the deduction threshold." +us,scenario_008,snap,grok-build-0.1,llm_error,asset_resource,False,"The model applied the ordinary $4,250 resource limit to the $69,500 bank balance. NJ broad-based categorical eligibility removes the SNAP asset test for this household, so the assets do not disqualify it." +us,scenario_008,snap,inkling,llm_error,thresholds_rates,False,"The model used an unsupported $1,806 maximum and a rounded $529 contribution, then annualized an approximate $1,276 monthly benefit. The exact calendar-year schedule and traced contribution produce $15,246.91." +us,scenario_008,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so it failed the required output contract before any substantive calculation could be assessed." +us,scenario_008,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly derived $1,763.92 net monthly income but applied the $1,789 maximum allotment to all twelve months. Calendar 2026 includes higher later-period maximum allotments near $1,829, raising the annual total to $15,246.91." +us,scenario_008,snap,minimax-m3,llm_error,asset_resource,False,"The model denied eligibility by applying a $3,750 asset limit to the household's bank balance. NJ categorical eligibility removes that resource test, so the $69,500 balance does not bar SNAP." +us,scenario_008,snap,qwen-3.7-max,llm_error,other,False,"The model first computed $15,391, then replaced it with $14,784 through an unspecified housing-assistance adjustment. Housing assistance does not justify that adjustment; the traced deductions and monthly allotment schedule yield $15,246.91." +us,scenario_008,snap,qwen3.8-max,llm_error,thresholds_rates,False,"The model used a gross-income threshold below the household's $2,576 monthly income and ignored NJ's higher categorical-eligibility threshold. The household's applicable gross threshold is $4,512.50, so it passes at 57% of FPG and remains eligible." +us,scenario_008,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply the ACA adult expansion pathway. At age 41 and 0.55 times FPL, the spouse is in New Jersey's under-65 MAGI adult category and falls below its 138% FPL limit." +us,scenario_008,spouse_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly found income below 138% FPL but invented a housing-assistance disqualification. Receipt of housing assistance does not bar Medicaid eligibility, and the spouse qualifies through the ACA adult expansion category without needing disability as the categorical basis." +us,scenario_008,spouse_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,The model treated the disability flag as automatic Medicare eligibility regardless of age. It failed to distinguish disability status from the qualifying SSDI entitlement and waiting-period conditions required for the under-65 pathway. +us,scenario_008,spouse_medicare_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly made disability alone sufficient for Medicare eligibility. At age 41, the spouse needs a separately established under-65 Medicare pathway, which the listed facts do not provide." +us,scenario_008,spouse_medicare_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model misrepresented PolicyEngine's eligibility logic by converting the spouse's disability flag directly into Medicare eligibility. The spouse fails the age-65 test, and no qualifying under-65 Medicare status is listed." +us,scenario_008,spouse_medicare_eligible,kimi-k3,llm_error,categorical_eligibility,False,"The model treated every disabled person under 65 as Medicare eligible. It omitted the distinct qualifying entitlement conditions for disability-based Medicare, leaving the 41-year-old spouse ineligible." +us,scenario_008,spouse_medicare_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model invented SSDI receipt and satisfaction of the 24-month entitlement period from disability and housing-assistance receipt. Housing assistance does not establish SSDI entitlement, and the prompt explicitly makes unlisted benefit receipt and statuses false." +us,scenario_008,spouse_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented a WIC “categorically eligible caretaker” pathway based on the spouse living with a one-year-old. WIC does not extend the child's categorical status to a parent; the spouse must herself be pregnant, breastfeeding, or postpartum." +us,scenario_008,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated motherhood and the presence of young children as the spouse's categorical qualifier. A mother qualifies only through her own pregnancy, breastfeeding, or postpartum status, none of which is listed." +us,scenario_008,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model inferred breastfeeding or postpartum status from the presence of a one-year-old despite the instruction that unlisted statuses are false. It also mislabeled low income as adjunctive categorical eligibility; passing the income test does not supply the spouse's missing person-level WIC category. +us,scenario_008,spouse_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly used disability as a WIC categorical pathway. Disability is not one of WIC's person categories, and the spouse has no listed pregnancy, breastfeeding, or postpartum status." us,scenario_008,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model counted blindness and disability as two separate $1,000 exemptions even though New Jersey provides one combined blind-or-disabled exemption, overstating exemptions by $1,000. It also deducted 18% of pre-subsidy rent as property tax despite no eligible tenant-paid property tax amount being supplied." -us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly invoked a standard deduction, pension exclusion, property-tax deduction, and the federal American Opportunity Credit in computing New Jersey liability. The specified income less the $12,000 New Jersey exemptions leaves $18,915 taxable rather than zero." -us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model double-counted the spouse's blindness and disability as separate exemptions, producing $17,915 instead of $18,915 of taxable income. It then invented unspecified nonrefundable credits to erase the remaining tax instead of applying the 1.4% rate." -us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model misapplied New Jersey's $20,000 joint-filer gross-income filing threshold as though exemptions or credits could be combined with it. Gross income is $30,915, so the threshold does not eliminate liability, and $18,915 remains taxable." -us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model assigned $3,000 of ordinary exemptions to the two spouses instead of the $2,000 joint-filer exemption and thereby overstated total exemptions by $1,000. The correct $12,000 total leaves $18,915 taxable and produces $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model counted separate $1,000 exemptions for the spouse's blindness and disability instead of one combined blind-or-disabled exemption. It then fabricated a dependent-sensitive low-income phaseout and rounding rule that reduced a positive statutory liability to zero." -us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used $30,632 instead of the stated $30,915 of New Jersey income and allowed only $9,000 of exemptions. It omitted the $2,000 joint-filer exemption and $1,000 blind-or-disabled exemption, which together bring total exemptions to $12,000." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model correctly derived $18,915 of taxable income and approximately $264.81 of tax, then wrongly subtracted a $50 tenant property-tax credit from an output defined before refundable credits. That credit does not reduce this pre-refundable-credit liability." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The zero answer treats the household's income and exemptions as eliminating the tax base. Subtracting the allowable $12,000 exemptions from $30,915 leaves $18,915 taxable, not zero." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model first understated preliminary tax by using an extra $1,000 exemption, then subtracted a $50 tenant property-tax credit from the pre-refundable-credit output. The applicable derivation uses $12,000 of exemptions and does not subtract refundable credits, yielding $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model allowed $14,000 of exemptions by assigning $3,000 to joint filers and separate $1,000 exemptions for blindness and disability. New Jersey instead allows a $2,000 joint exemption and one $1,000 blind-or-disabled exemption, for $12,000 total." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model correctly computed $264.81 from $18,915 of taxable income, then improperly subtracted a $50 property-tax credit. Refundable credits are excluded from state income tax before refundable credits." +us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the spouse's blindness and disability as two separate $1,000 exemptions instead of applying the single $1,000 blind-or-disabled exemption. It then improperly deducted 18% of rent despite the housing-assistance facts, rather than taxing $18,915 at 1.4%." +us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly imported a standard deduction, pension exclusion, property-tax relief, and the federal American Opportunity Credit into the New Jersey calculation. The listed income less the $12,000 of applicable exemptions leaves $18,915 taxable, not zero." +us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model double-counted blindness and disability, producing $13,000 rather than $12,000 of exemptions. It then erased its own positive tentative tax with unspecified nonrefundable credits even though none applies to this computation." +us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model misapplied New Jersey's $20,000 joint-filer gross-income filing threshold as an exemption or credit. Gross income is $30,915, so the household exceeds that threshold and owes 1.4% of its $18,915 taxable income." +us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model assigned $3,000 of ordinary exemptions to the two spouses and then added the disability exemption, yielding $13,000 total. The joint-filer regular exemption is $2,000 and the blind-or-disabled addition is $1,000, so total exemptions are $12,000." +us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the spouse's blind and disabled statuses as separate exemptions instead of one $1,000 blind-or-disabled exemption. It also invented a low-income phase-in and rounding rule that eliminates tax even though $18,915 remains taxable at 1.4%." +us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used $30,632 rather than the listed $30,915 of New Jersey income and allowed only $9,000 of exemptions. The correct exemptions are $12,000, leaving $18,915 entirely within the 1.4% bracket." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly derived $18,915 of taxable income and approximately $265 of tax, then improperly subtracted a $50 tenant credit. No such nonrefundable renter credit applies in this calculation, so the full $264.81 remains." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated low income and personal exemptions as eliminating the liability. The $12,000 of exemptions reduce $30,915 only to $18,915, which produces $264.81 of tax." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model first overstated exemptions, producing a preliminary tax near $251 rather than $264.81, and then improperly subtracted a $50 tenant credit. The applicable exemption total is $12,000 and no tenant credit reduces this output." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model allowed $14,000 of exemptions by overstating the joint-filer exemption and separately counting blindness and disability. Only $12,000 applies, leaving $18,915 taxable rather than $16,915." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly calculated $264.81 from $18,915 of taxable income, then improperly subtracted a $50 property-tax credit. That credit does not apply to this pre-refundable-tax computation." us,scenario_008,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that exemptions and credits wipe out the tax without calculating them. The allowable $12,000 exemptions leave $18,915 taxable, and no applicable nonrefundable credit eliminates the resulting $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The zero estimate assumes unspecified deductions and credits eliminate the liability. The stated facts produce $18,915 of taxable New Jersey income after exemptions and therefore $264.81 of tax." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model invented a $181.70 medical deduction. The listed over-the-counter expenses do not create that deduction, so taxable income remains $18,915 and the 1.4% tax is $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model subtracted the New Jersey child tax credit from an output expressly measured before refundable credits. The exemptions leave $18,915 taxable, and the refundable child credit does not reduce the resulting $264.81 here." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model reduced liability using unsupported medical deductions and a renter property-tax credit. Neither adjustment enters this pre-refundable-credit derivation, which taxes $18,915 at 1.4%." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $250.81 answer implies taxable income of $17,915, reflecting an extra $1,000 exemption. Blindness and disability generate one combined $1,000 exemption, so total exemptions are $12,000 and taxable income is $18,915." -us,scenario_008,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied a New Jersey standard deduction and unspecified credits to eliminate liability. New Jersey's applicable exemptions total $12,000, leaving $18,915 taxable." -us,scenario_008,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated income as $30,632 and treated the six dependents as only $6,000 of exemptions. Their dependent exemptions total $9,000, and adding the $2,000 joint and $1,000 blind-or-disabled exemptions yields $12,000." -us,scenario_008,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model omitted the $1,000 blind-or-disabled exemption, leaving $19,915 instead of $18,915 taxable. It also applied 1.75% to the entire amount even though joint taxable income below $20,000 is taxed at 1.4%." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model assumed unspecified exemptions and credits wipe out the tax. The specified New Jersey exemptions leave $18,915 taxable, and no applicable nonrefundable credit reduces the resulting $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model reduced the liability to zero using unspecified deductions and credits. The applicable $12,000 of exemptions leave a positive $18,915 tax base taxed at 1.4%." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $181.70 of medical expenses from New Jersey taxable income. The listed over-the-counter expenses do not create that deduction, so taxable income remains $18,915 and tax is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model improperly used the New Jersey child tax credit to offset this before-refundable-credits liability. The applicable child credit is refundable and excluded from this output, leaving $264.81 before refundable credits." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied medical deductions and a nonrefundable renter property-tax credit that do not reduce this calculation. The applicable exemptions alone reduce income to $18,915, producing $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $250.81 answer corresponds to taxable income of $17,915, reflecting $13,000 of exemptions. New Jersey provides one $1,000 blind-or-disabled exemption rather than separate blindness and disability amounts, so exemptions total $12,000." +us,scenario_008,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model invented a New Jersey standard deduction and unspecified credits that reduce tax to zero. New Jersey's applicable exemptions leave $18,915 taxable and a $264.81 liability." +us,scenario_008,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model substituted a $30,632 federal-AGI figure for the $30,915 of listed New Jersey income and allowed only $9,000 of exemptions. Total exemptions are $12,000, keeping all $18,915 of taxable income in the 1.4% bracket." +us,scenario_008,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model omitted the $1,000 blind-or-disabled exemption and then taxed the entire base at 1.75%. Total exemptions are $12,000, and the resulting $18,915 is taxed entirely at 1.4%." +us,scenario_008,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used only $10,000 of exemptions and improperly deducted 18% of rent. The applicable exemptions total $12,000, while the renter deduction does not apply here, leaving $18,915 taxable." us,scenario_008,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. -us,scenario_008,state_income_tax_before_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model correctly derived $18,915 of taxable income and $264.81 of tax, then wrongly subtracted a $50 renter property-tax credit. That refundable credit is excluded from this before-refundable-credits output." -us,scenario_008,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used $30,632 instead of $30,915 and omitted the $1,000 blind-or-disabled exemption. Correct income of $30,915 less $12,000 of exemptions leaves $18,915 taxable." -us,scenario_008,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model valued each dependent exemption at $1,000 instead of $1,500 and treated disability as potentially separate from the combined blind-or-disabled exemption. It then invented a nonrefundable low-income reduction to zero even though $30,915 less the correct $12,000 exemptions produces $264.81 of tax." -us,scenario_008,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model incorrectly phased the federal EITC down to $6,110 instead of using the traced $8,231 amount, and it limited the NJ CTC to two children under age six instead of the four age-eligible dependents receiving $2,000. It also omitted the $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that New Jersey has no refundable credits. This household receives a refundable NJ EITC of $3,292.40, NJ CTC of $2,000, and renter Property Tax Credit of $50." -us,scenario_008,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The submitted $9,492 does not follow from the model's own stated components of roughly $2,880 of NJ EITC and $1,000 of NJ CTC. It also understated the traced NJ EITC, omitted three of the four CTC-eligible dependents, and omitted the $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model applied an obsolete under-six conception of the NJ CTC and estimated only $1,200 instead of the traced $2,000 for four age-eligible dependents. It also failed to separately include the $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly declared the NJ CTC inapplicable at this income and approximated the NJ EITC as $3,080 rather than $3,292.40. It also omitted the $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model used the wrong federal EITC phaseout start and reduced the federal credit to $6,716 instead of the traced $8,231, producing too little NJ EITC. It also restricted the NJ CTC to two children under six rather than four age-eligible dependents and omitted the $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model estimated rather than computing the NJ EITC and treated only the two children under six as NJ CTC qualifiers rather than the four age-eligible dependents. It also omitted the $50 renter Property Tax Credit, so its rounded $5,800 does not match the program-level calculation." -us,scenario_008,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model used $7,787 rather than $8,231 as the federal EITC base and treated the NJ EITC as the entire state refundable-credit total. It omitted the $2,000 NJ CTC and $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model understated the federal EITC base and applied a two-child, $800-per-child NJ CTC rule instead of the traced $2,000 credit for four age-eligible dependents. It also omitted the $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model failed to identify all three refundable New Jersey programs. The household receives $3,292.40 from the NJ EITC, $2,000 from the NJ CTC, and $50 from the renter Property Tax Credit." -us,scenario_008,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model limited the NJ CTC to two children under six and awarded $1,600 instead of the traced $2,000 for four age-eligible dependents. It also omitted the $50 renter Property Tax Credit and rounded the NJ EITC below $3,292.40." -us,scenario_008,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model incorrectly reduced the federal EITC to $7,128 rather than $8,231 and therefore understated the 40% NJ EITC. It also used only two CTC-eligible children and omitted the $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model understated the NJ EITC by $72.80 and awarded only $1,600 of NJ CTC instead of $2,000 for four age-eligible dependents. It also omitted the $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value for state_refundable_credits, so the required output was missing." -us,scenario_008,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable New Jersey credit was indicated. The household qualifies for the NJ EITC, NJ CTC, and renter Property Tax Credit totaling $5,342.40." -us,scenario_008,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of separately stated credit inputs as disqualifying even though income, children, renter status, and age establish eligibility. It consequently omitted the $3,292.40 NJ EITC, $2,000 NJ CTC, and $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"The model correctly included the $3,292.40 NJ EITC and $2,000 NJ CTC but omitted the $50 refundable Property Tax Credit. The household qualifies for that credit as renters with a spouse age 42." -us,scenario_008,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model used $8,046 instead of $8,231 as the federal EITC base and included only the resulting NJ EITC. It omitted the $2,000 NJ CTC and $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model correctly computed the NJ EITC but limited the NJ CTC to $1,600 for two children under six instead of $2,000 for four age-eligible dependents. It also omitted the $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The submitted total equals the correct $3,292.40 NJ EITC plus only $1,600 of NJ CTC, reflecting the erroneous two-under-six shortcut rather than the traced $2,000 for four age-eligible dependents. It also omits the $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly stated that no state refundable credits apply. New Jersey provides this household $3,292.40 of EITC, $2,000 of CTC, and a $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model used the wrong federal EITC base and an obsolete $400-per-child, under-six NJ CTC rule, yielding only $800 rather than $2,000 for four age-eligible dependents. It also omitted the $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model used $7,901 instead of $8,231 as the federal EITC base and incorrectly treated the NJ EITC as the only refundable state credit. It omitted the $2,000 NJ CTC and $50 renter Property Tax Credit." -us,scenario_008,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value for state_refundable_credits, so the required output was missing." -us,scenario_008,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,"The model correctly included the $3,292.40 NJ EITC and $2,000 NJ CTC but omitted the $50 refundable Property Tax Credit available to renters when a spouse is at least age 42." -us,scenario_008,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no New Jersey refundable credits apply. The household qualifies for three components totaling $5,342.40: NJ EITC, NJ CTC, and the renter Property Tax Credit." -us,scenario_008,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model incorrectly treated a married couple's federal EITC as phased out near $30,800 even though the traced federal EITC is $8,231, and it counted only one under-six child while ignoring the broader NJ CTC age rule that covers four dependents. Housing assistance does not eliminate these refundable credits, tax liability need not be positive, and the model also omitted the $50 renter Property Tax Credit." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived the complete $8,348 calculation correctly, then submitted $3,455 instead. Its submitted value contradicts every stated computation step and is an unexplained final-answer substitution." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the listed $8,139 employer-sponsored insurance premium from wages even though the prompt reports gross annual wages and does not identify an employee pre-tax wage deduction. It also used a $30,000 rather than $32,200 standard deduction and then reduced its own approximately $8,600 tax calculation to $2,950 without any credit or computation supporting that reduction." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's stated method produced $8,336 using a slightly wrong $32,300 standard deduction, but it submitted $4,880. No deduction or nonrefundable credit in its reasoning accounts for the $3,456 reduction, so the submitted value is an unexplained final-answer substitution." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly found $25,500 of taxable Social Security and calculated tax near $8,349, but submitted $5,732. Nothing in its derivation produces that value, making the error an unsupported replacement of its computed result." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly computed tax of about $8,300 after finding the proper taxable Social Security amount, yet submitted $4,989. Its claimed rounding cannot turn $8,300 into $4,989, and it identified no applicable credit or deduction producing the reduction." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used projected parameters—approximately $30,700 for the standard deduction and $24,450 for the 10% bracket ceiling—instead of the applicable 2026 values of $32,200 and $24,800. Applying the actual parameters to $105,900 of AGI yields $73,700 of taxable income and $8,348 of tax." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model included only about $16,150 of Social Security in AGI even though the provisional-income formula reaches the 85% cap, making $25,500 taxable. It then applied unspecified ""minor credit adjustments"" despite identifying no credit, reducing its own $7,178 calculation to $6,350." -us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the spouse's $60,000 gross wages by the $8,139 employer-sponsored insurance premium, producing AGI of $97,761 instead of $105,900. It also used estimated $30,600 and $24,400 deduction and bracket parameters rather than the applicable $32,200 standard deduction and $24,800 bracket ceiling." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the ESI premium from gross wages and invented mortgage interest, state tax, and deductible medical amounts that the prompt did not supply. It also applied a TCJA-sunset regime with personal exemptions and 15% brackets instead of the applicable 2026 standard deduction and 10%/12% brackets." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's stated $80,360 base conflates gross income, taxable Social Security, and a $10,040 deduction without a valid federal AGI calculation. The correct sequence gives $25,500 of taxable Social Security, $105,900 of AGI, a $32,200 standard deduction, and $73,700 of taxable income." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $8,139 ESI premium from the spouse's gross wages, understating AGI by that amount. It compounded this with an inapplicable TCJA-expiration combination of deductions, personal exemptions, and 15% rates rather than the governing 2026 $32,200 standard deduction and 10%/12% brackets." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied reverted pre-TCJA rules, including a $16,256 standard deduction, personal exemptions, and a 15% marginal bracket, rather than the applicable 2026 parameters. It also improperly subtracted the ESI premium from the expressly stated gross wages." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly treated the listed $8,139 employer-sponsored insurance premium as a pre-tax reduction from the spouse's $60,000 gross wages, producing $97,761 rather than $105,900 of AGI. The correct taxable-income calculation uses the full wages and the $32,200 standard deduction." -us,scenario_009,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly calculated AGI but substituted estimated parameters: a $30,750 standard deduction and $24,450 first-bracket ceiling. The applicable $32,200 deduction and $24,800 ceiling produce $73,700 of taxable income and $8,348 of tax." -us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly declared the household below the taxable threshold without calculating taxable Social Security, AGI, or the standard deduction. Wages, pension income, and $25,500 of taxable Social Security produce $105,900 of AGI and $73,700 of taxable income, not zero taxable income." -us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked itemized health-related deductions without calculating the 7.5% of AGI medical-expense floor or comparing itemized deductions with the $32,200 standard deduction. The standard deduction is larger than the $3,207.96 itemized amount, leaving $73,700 taxable and $8,348 of tax." -us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model stated that taxable income was $26,150 but then submitted $3,922 despite claiming no nonrefundable credits; neither figure follows from the listed income and standard deduction. The proper derivation yields $105,900 of AGI, $73,700 of taxable income, and $8,348 of tax." -us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly found $105,900 of AGI but applied an inapplicable post-sunset regime with a $17,018 standard deduction, personal exemptions, and a 15% bracket. The governing 2026 calculation instead subtracts the $32,200 standard deduction and applies the 10% and 12% joint brackets." -us,scenario_009,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used pre-TCJA parameters—$15,700 standard deduction, personal exemptions, and a 15% marginal rate—that do not govern this 2026 calculation. Using the $32,200 joint standard deduction and applicable 10%/12% brackets yields $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly calculated $105,900 of AGI but used estimated values of $30,600 for the standard deduction and $24,400 for the 10% bracket ceiling. The applicable values are $32,200 and $24,800, producing $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used approximate parameters of a $30,900 standard deduction and a $24,500 first-bracket ceiling. The applicable $32,200 deduction and $24,800 ceiling reduce taxable income to $73,700 and tax to $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model used outdated projected parameters and calculated $8,577, then submitted $8,077.50 with no intervening deduction or credit. The correct 2026 parameters are a $32,200 standard deduction and $24,800 first-bracket ceiling, and the unexplained $499.50 final reduction has no basis in its reasoning." +us,scenario_008,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly derived $18,915 of taxable income and $264.81 of tax, then improperly subtracted a $50 renter property-tax credit. No such credit reduces this output." +us,scenario_008,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used $30,632 instead of the listed $30,915 of income and omitted the $1,000 blind-or-disabled exemption. Correct income of $30,915 less $12,000 of exemptions yields $18,915 taxable." +us,scenario_008,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model valued each dependent exemption at $1,000 instead of $1,500 and treated disability as potentially separate from blindness. It then invented a nonrefundable low-income reduction to zero; the actual $12,000 exemption total leaves $18,915 taxable." +us,scenario_008,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that exemptions and deductions reduce New Jersey taxable income to zero. The applicable exemptions total only $12,000 against $30,915 of income, leaving $18,915 taxable at 1.4%." +us,scenario_008,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model applied the federal EITC phaseout at the wrong income threshold, reducing the federal credit from $8,231 to $6,110. It also used roughly $30,000 of income to award only $800 of NJ CTC instead of applying $18,915 of NJ taxable income to four qualifying children for $2,000, and it omitted the $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that New Jersey has no refundable state credits. This household receives the refundable NJ EITC, NJ CTC, and renter property-tax credit." +us,scenario_008,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The submitted $9,492 does not follow the model's own stated components of approximately $2,880 of NJ EITC and $1,000 of NJ CTC. It also omitted the four-child $2,000 NJ CTC derivation and the $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model used federal AGI near $30,500 to reduce the NJ CTC and counted only the two children under age six. NJ taxable income is $18,915 and four dependent children qualify, producing $2,000 of CTC; the model also failed to separately add the $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,The model understated the federal EITC base and incorrectly declared the NJ CTC inapplicable. It also omitted the $50 renter property-tax credit available because the household rents and one spouse is at least age 42. +us,scenario_008,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model applied an EITC phaseout using the wrong threshold, reducing the federal EITC to $6,716 rather than $8,231. It also based the NJ CTC on federal AGI and two children under six instead of $18,915 of NJ taxable income and four qualifying children, and omitted the $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model used loose EITC and CTC ranges rather than the applicable amounts, overstating the NJ EITC while not deriving the four-child $2,000 NJ CTC from $18,915 of taxable income. It also omitted the separate $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model used an understated federal EITC of $7,787 instead of $8,231. More importantly, it counted only the NJ EITC and omitted the $2,000 NJ CTC and $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model understated the federal EITC and used gross income near $30,000 to place two young children in an $800 CTC tier. The calculation instead uses $18,915 of NJ taxable income, awards $2,000 for four qualifying children, and adds the $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model failed to identify all three refundable New Jersey credits. The household qualifies for the NJ EITC, NJ CTC, and renter property-tax credit." +us,scenario_008,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model used only two children under age six and an $800-per-child tier for the NJ CTC. At $18,915 of NJ taxable income, four dependent children generate $2,000 of CTC, and the household also receives the omitted $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model incorrectly phased the federal EITC down to $7,128 and therefore understated the NJ EITC. It also awarded CTC to only two children at the wrong income tier and omitted the $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model understated the NJ EITC rather than taking 40% of the $8,231 federal EITC, and it limited the NJ CTC to $1,600. The correct CTC is $2,000 for four qualifying children at $18,915 of NJ taxable income, plus a $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no value or explanation for state_refundable_credits. +us,scenario_008,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated the facts as establishing no refundable New Jersey credit. The earnings, dependent children, and renter status establish eligibility for the NJ EITC, NJ CTC, and renter property-tax credit." +us,scenario_008,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model overlooked the specific qualifying circumstances already provided: earned income and qualifying children for the NJ EITC, four qualifying dependents for the NJ CTC, and renter status with a spouse age 42 for the property-tax credit." +us,scenario_008,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"The model correctly computed the $3,292.40 NJ EITC and $2,000 NJ CTC but omitted the $50 refundable property-tax credit. The household qualifies for that credit as renters with one spouse age 42." +us,scenario_008,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model understated the federal EITC base as $8,046 instead of $8,231. It then omitted both the $2,000 NJ CTC for four qualifying children and the $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model correctly calculated the $3,292.40 NJ EITC but awarded only $1,600 of NJ CTC based on two children under six. The applicable calculation awards $2,000 for four qualifying children at $18,915 of NJ taxable income and adds the omitted $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The submitted total implies the correct $3,292.40 NJ EITC plus only $1,600 of NJ CTC. The correct CTC is $2,000 for four qualifying children, and the model also omitted the $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no state refundable credits apply. New Jersey awards this household an EITC, a child tax credit, and a renter property-tax credit." +us,scenario_008,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model used gross income to place only two children in a $400 NJ CTC tier. The credit calculation uses $18,915 of NJ taxable income and four qualifying children for a $2,000 CTC; it also omitted the $50 renter property-tax credit and used the wrong federal EITC amount." +us,scenario_008,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model used an understated federal EITC of $7,901 and counted only the resulting NJ EITC. It omitted the $2,000 NJ CTC and the $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model used the wrong federal EITC amount and treated the NJ EITC as the entire refundable-credit total. It omitted the $2,000 NJ CTC and $50 renter property-tax credit." +us,scenario_008,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for state_refundable_credits. +us,scenario_008,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,"The model correctly calculated the $3,292.40 NJ EITC and $2,000 NJ CTC but omitted the $50 refundable renter property-tax credit. A spouse is age 42 and the household rents, satisfying that credit's stated conditions." +us,scenario_008,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no New Jersey refundable credits apply. The household qualifies for all three components: NJ EITC, NJ CTC, and the renter property-tax credit." +us,scenario_008,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model incorrectly treated roughly $30,800 of earned income as eliminating the federal EITC, even though the applicable computation yields $8,231. It also counted only one young child, wrongly treated housing assistance or zero tax liability as disqualifying refundable credits, and omitted the renter property-tax credit." +us,scenario_008,state_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model invented an NJ EITC phaseout threshold below this household's income and substituted a nonexistent $100 minimum credit. The applicable federal EITC is $8,231, producing a $3,292.40 NJ EITC, with another $2,000 of NJ CTC and $50 of renter property-tax credit." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived $73,700 of taxable income and $8,348 of tax, then submitted $3,455 instead. Its submitted value directly contradicts every computation in its explanation." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the spouse's $8,139 employer-sponsored insurance premium from gross wages and used a $30,000 standard deduction instead of $32,200. It then submitted $2,950 despite its own intermediate estimate of about $8,600 and identified no credit that could produce the reduction." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's only parameter error was using a $32,300 standard deduction, which led it to calculate $8,336 rather than $8,348. It then submitted $4,880, a value unrelated to its stated taxable-income and bracket calculation." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly capped taxable Social Security at $25,500 and calculated tax near $8,349 using its estimated parameters. It then submitted $5,732 without any deduction, credit, or tax computation supporting that amount." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived $105,900 of AGI and tax of about $8,300, with no nonrefundable credits. It nevertheless submitted $4,989, which does not follow from its stated brackets or deductions." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used projected parameters—a $30,700 standard deduction and a $24,450 top of the 10% bracket—instead of the exact 2026 values of $32,200 and $24,800. That inflated taxable income and produced $8,535 rather than $8,348." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model included only about $16,150 of the $30,000 Social Security benefit even though the statutory calculation reaches the 85% cap of $25,500. It also invented unspecified “minor credit adjustments” and submitted $6,350 despite its own bracket calculation yielding $7,178." +us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the spouse's $60,000 gross wages by the listed $8,139 employer-sponsored insurance premium, producing AGI of $97,761 instead of $105,900. It also used a $30,600 standard deduction and a $24,400 bracket threshold rather than the exact 2026 parameters." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly treated 2026 as a TCJA-sunset year, applied personal exemptions and 10%/15% brackets, and invented mortgage interest, state tax, and deductible medical amounts from inputs that did not supply them. It also improperly netted the $8,139 ESI premium from wages." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model did not construct AGI or taxable income under federal rules: it treated the full Social Security amount as income and described subtracting a $10,040 “standard deduction” inside an $80,360 gross adjusted income figure. The required derivation instead yields $105,900 of AGI and $73,700 of taxable income." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by the $8,139 ESI premium and assumed TCJA expiration, combining an obsolete standard deduction with personal exemptions and 10%/15% rates. Current 2026 law uses $105,900 of AGI, a $32,200 standard deduction, and the 10%/12% brackets applicable here." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied reverted pre-TCJA deductions, personal exemptions, and 10%/15% brackets instead of the operative 2026 $32,200 standard deduction and 10%/12% brackets. It also incorrectly removed the $8,139 ESI premium from the spouse's stated gross wages." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the spouse's $8,139 employer-sponsored insurance premium from the stated $60,000 of gross wages, lowering AGI to $97,761. Gross wages remain $60,000 under the supplied inputs, so AGI is $105,900 before the $32,200 standard deduction." +us,scenario_009,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly calculated AGI and taxable Social Security but substituted projected parameters: a $30,750 standard deduction and $24,450 10% bracket cutoff. The exact 2026 values are $32,200 and $24,800, producing $73,700 of taxable income and $8,348 of tax." +us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly declared the household below the taxable threshold without computing taxable Social Security, AGI, or the standard deduction. Wages, pension income, and $25,500 of taxable Social Security produce $73,700 of taxable income, so the liability is not zero." +us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The submitted $2,757 reflects unsupported health-related itemized deductions or other reductions. Itemized deductions total only $3,207.96, so the household takes the $32,200 standard deduction and owes $8,348 with no nonrefundable credits." +us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model stated that taxable income was $26,150, omitting most of the $105,900 AGI after the applicable $32,200 standard deduction. It then submitted $3,922 despite saying no nonrefundable credits applied and without a bracket calculation connecting that value to its stated taxable income." +us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model wrongly assumed a 2026 TCJA sunset and applied a small standard deduction, personal exemptions, and a 15% second bracket. The operative rules instead provide a $32,200 MFJ standard deduction and apply 10% and 12% rates to $73,700 of taxable income." +us,scenario_009,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model wrongly replaced the operative 2026 rules with pre-TCJA personal exemptions, a $15,700 standard deduction, and a 15% second bracket. The correct $32,200 standard deduction leaves $73,700 taxable under the 10% and 12% brackets." +us,scenario_009,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model improperly treated the $8,139 employer-sponsored insurance premium as a pre-tax reduction from the spouse's stated gross wages. Restoring wages to $60,000 raises AGI from its $97,761 figure to $105,900 and taxable income to $73,700." +us,scenario_009,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly calculated $105,900 of AGI but used estimated 2026 parameters: a $30,600 standard deduction and a $24,400 10% bracket cutoff. The exact $32,200 deduction and $24,800 cutoff produce $8,348." +us,scenario_009,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used an estimated $30,900 standard deduction and $24,500 10% bracket cutoff instead of the exact 2026 values of $32,200 and $24,800. Those exact parameters reduce taxable income to $73,700 and tax to $8,348." +us,scenario_009,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model used outdated projected parameters and calculated $8,577 from them, but submitted $8,077.50. No stated credit or subsequent calculation supports the $499.50 reduction from its own computed tax." +us,scenario_009,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model included only $4,000 of taxable Social Security instead of the 85% cap of $25,500 and invented a $1,050 CDCC despite no dependent or care expense being listed. Its submitted $12,110.90 also contradicts its stated $5,930.25 pre-credit tax and subtraction of that purported credit." us,scenario_009,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_009,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model inferred an unlisted SSDI/disability status from Social Security retirement income and then applied the under-65 disability Medicare pathway. The benchmark facts list age 57 and do not list SSDI receipt, disability, ESRD, ALS, or Medicare enrollment, so the age-65 Medicare condition is not met and no under-65 pathway applies." -us,scenario_009,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model's explanation correctly states that the 57-year-old head is below the standard Medicare eligibility age and is not eligible, but it submitted the opposite binary value. This is a parse/answer consistency error: the reasoning supports value 0, while the output value 1 marks Medicare eligibility." +us,scenario_009,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model inferred an unlisted disability and SSDI history from Social Security retirement income, contradicting the instruction that unlisted statuses are false. At age 57 with no disability, ALS, or end-stage renal disease status, the head does not qualify for Medicare." +us,scenario_009,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model's reasoning correctly concluded that the 57-year-old head is below age 65 and not eligible, but it submitted value = 1 instead of value = 0. This is an internal answer-to-value contradiction." us,scenario_009,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_009,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the $8,139 employer-sponsored insurance premium from Social Security and Medicare wages. Applying 6.2% and 1.45% to the full $60,000 produces $4,590." -us,scenario_009,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed $8,139 employer-sponsored insurance premium as a pre-tax employee payroll deduction that reduces FICA wages, despite no such fact being provided. FICA applies to the full $60,000 wage amount, yielding $4,590." -us,scenario_009,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages from $60,000 to $51,861 by subtracting the employer-sponsored insurance premium. Social Security and Medicare taxes on the unreduced wages total $3,720 plus $870, or $4,590." -us,scenario_009,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model mislabeled $51,861 as pre-tax FICA wages by deducting the $8,139 insurance premium without an input establishing a FICA-exempt employee contribution. The full $60,000 is subject to the 7.65% employee rate, producing $4,590." -us,scenario_009,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly assumed the employer-sponsored insurance premium was deductible from the payroll-tax base. Applying the employee Social Security and Medicare rates to the full $60,000 yields $4,590." -us,scenario_009,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model used the combined employer-and-employee FICA rates of 12.4% for Social Security and 2.9% for Medicare even though the requested output includes only employee-side payroll tax. The employee rates are 6.2% and 1.45% on $60,000, totaling $4,590, and the model's stated rates would produce $9,180 rather than its submitted $13,663." +us,scenario_009,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model incorrectly deducted the spouse's $8,139 employer-sponsored insurance premium from FICA wages. The payroll-tax base is the full $60,000, producing $3,720 of Social Security tax plus $870 of Medicare tax." +us,scenario_009,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed $8,139 employer-sponsored insurance premium as a pre-tax reduction of Social Security and Medicare wages. Applying 7.65% to the full $60,000 wage amount yields $4,590." +us,scenario_009,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages from $60,000 to $51,861 using the employer-sponsored insurance premium. The listed premium does not reduce the payroll-tax base, so the employee tax is $4,590." +us,scenario_009,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model mislabeled $51,861 as pre-tax FICA wages by subtracting the $8,139 insurance premium. Social Security and Medicare taxes instead apply to all $60,000 of wages, yielding $4,590." +us,scenario_009,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly deducted the employer-sponsored insurance premium when determining taxable wages for payroll tax. The correct FICA base is $60,000, to which the combined 7.65% employee rate produces $4,590." +us,scenario_009,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model used the combined employee-and-employer rates of 12.4% for Social Security and 2.9% for Medicare even though the requested output includes only employee-side tax. Its stated rates also do not arithmetically produce $13,663 from $60,000; the employee rates of 6.2% and 1.45% yield $4,590." +us,scenario_009,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the spouse's employer-sponsored insurance premium from wages before calculating FICA. Applying the employee Social Security and Medicare rates to the full $60,000 produces $4,590, with no Additional Medicare Tax or North Carolina employee payroll tax." us,scenario_009,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_009,spouse_medicare_eligible,gpt-5.4-mini,llm_error,health_coverage,False,"The model's explanation applied the correct Medicare rule and stated that the 57-year-old spouse is not eligible, but it submitted `value = 1`, which encodes Yes. This is an output contract failure: the numeric value contradicts the model's own eligibility explanation." +us,scenario_009,spouse_medicare_eligible,gpt-5.4-mini,llm_error,other,False,"The model correctly stated that the 57-year-old spouse is not Medicare eligible but reversed the required binary encoding by submitting 1. The prompt requires 0 when the explanation says “not eligible,” so the supported value is 0." us,scenario_009,spouse_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_009,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model explicitly derived $54,900 of taxable income and correctly computed $54,900 × 3.99% = $2,190.51, then submitted $2,726 instead. Its final value does not follow its own completed calculation." -us,scenario_009,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented adjustments for health-insurance premiums and unspecified dependent, education, and other nonrefundable credits despite the prompt setting unlisted facts to false or zero. With no applicable nonrefundable credits, $54,900 is taxable and produces $2,190.51 rather than zero." -us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an obsolete 4.25% rate instead of North Carolina's 3.99% rate for 2026. It also submitted $2,891 even though neither its stated 4.25% calculation nor its comparison at 4.5% produces that amount." -us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly derived $54,900 of taxable income and $2,191 of tax at 3.99%, but then submitted $2,974 after an unsupported adjustment. North Carolina's Social Security subtraction was already fully reflected in the $54,900 base and required no further adjustment." -us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly identified $54,900 of taxable income and the 3.99% rate, then discarded that computation and applied the rate to an invented $71,600 base. The trace contains no pre-deduction adjustment that produces that base; the calculation is $54,900 × 3.99% = $2,190.51." -us,scenario_009,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the explicitly reported $60,000 of gross wages by the spouse's separately listed $8,139 employer-sponsored insurance premium. It also invented mortgage interest by applying an assumed 7% rate to the mortgage balance, even though mortgage interest was unlisted and therefore zero; the applicable deduction is the $25,500 standard deduction." -us,scenario_009,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model incorrectly discussed subtracting the full $30,000 Social Security benefit rather than the $25,500 included in federal AGI and invented mortgage interest and property taxes from a mortgage balance. More decisively, it submitted $3,200 even though every calculation in its explanation produced roughly $1,913 to $2,142." -us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced gross wages by the separately reported employer-sponsored insurance premium, producing an understated $97,761 federal AGI, and then used an incorrect $26,500 joint standard deduction. Federal AGI is $105,900 and the North Carolina joint standard deduction is $25,500, leaving $54,900 taxable." -us,scenario_009,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the spouse's $8,139 employer-sponsored insurance premium from the explicitly reported $60,000 gross wage input. That understates North Carolina income by $8,139; after the Social Security subtraction and $25,500 standard deduction, taxable income is $54,900." -us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used a 4.5% North Carolina rate instead of the 2026 rate of 3.99% and an unsupported taxable-income estimate of $60,800. The specified income and deductions yield $54,900 of taxable income and $2,190.51 of tax." -us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model's $97,761 starting AGI reflects an improper $8,139 reduction of the explicitly reported $60,000 gross wages for employer-sponsored insurance premiums. The correct federal AGI is $105,900, so North Carolina taxable income is $54,900 rather than $46,761." -us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model both understated federal AGI at $97,761 by netting employer-sponsored premiums from gross wages and used an incorrect $29,250 North Carolina joint standard deduction. The applicable figures are $105,900 of federal AGI and a $25,500 standard deduction, producing $54,900 of taxable income." -us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model's $97,761 federal AGI improperly nets the spouse's $8,139 employer-sponsored insurance premium from the explicitly reported $60,000 gross wages. Retaining the full wage input raises North Carolina taxable income from $46,761 to $54,900." -us,scenario_009,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model skipped the required state taxable-income computation and treated the absence of listed credits as support for zero tax. In fact, no applicable nonrefundable credit offsets the $2,190.51 generated by $54,900 of taxable income at 3.99%." -us,scenario_009,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and credits eliminate the liability without identifying any such deduction or credit. The listed facts yield $54,900 of North Carolina taxable income, and no nonrefundable state credit reduces the resulting $2,190.51 to zero." -us,scenario_009,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used the obsolete 4.5% rate instead of North Carolina's 3.99% rate for 2026 and did not derive the taxable-income base. The correct base is $54,900, producing $2,190.51." -us,scenario_009,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used a $21,000 joint standard deduction instead of $25,500, overstating taxable income by $4,500. Applying 3.99% to the correct $54,900 base yields $2,190.51." -us,scenario_009,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required structured-output contract before any substantive tax calculation could be evaluated. -us,scenario_009,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $25,500 of federally taxable Social Security that North Carolina excludes and used an incorrect approximate standard deduction and rate. The required subtractions from $105,900 leave $54,900 taxable at 3.99%." -us,scenario_009,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model ultimately derived the correct $54,900 taxable-income base but applied an obsolete 4.5% rate. North Carolina's 2026 rate is 3.99%, so the liability is $2,190.51 rather than $2,470.50." +us,scenario_009,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived $54,900 of taxable income and $2,190.51 of tax, then submitted the unrelated value $2,726. Its numeric output contradicts its completed calculation." +us,scenario_009,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model invented dependent, education, and other nonrefundable credits despite no qualifying dependents, students, or credit facts. No such credits eliminate the $2,190.51 North Carolina liability." +us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an obsolete estimated 4.25% rate instead of North Carolina's 2026 rate of 3.99%, then submitted $2,891 even though its own stated rate calculation produced about $2,333. The correct calculation is $54,900 × 3.99% = $2,190.51." +us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly calculated $54,900 of taxable income and approximately $2,191 of tax, but then submitted $2,974 after an unsupported adjustment. Taxable Social Security is included in federal AGI and deducted once for North Carolina, with no further adjustment." +us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model identified the correct $54,900 taxable income and 3.99% rate, then discarded that calculation and asserted an unsupported $71,600 tax base. North Carolina applies 3.99% to $54,900, not $71,600." +us,scenario_009,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the listed $60,000 gross annual wages by $8,139 of employer-sponsored insurance premiums, even though the wage input is already the full tax-year amount used in AGI. It also fabricated mortgage interest from the mortgage balance and an assumed 7% rate; no mortgage-interest payment was listed, so the $25,500 standard deduction applies." +us,scenario_009,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's submitted $3,200 does not follow any of its stated calculations, which ranged from about $1,913 to $2,142. It also subtracted the full $30,000 Social Security benefit instead of the $25,500 included in federal AGI and speculated about unlisted mortgage interest and property taxes." +us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced wages by the employer-sponsored premium and therefore started from $97,761 rather than $105,900 of federal AGI. It also used a $26,500 joint standard deduction instead of $25,500; the correct taxable income is $54,900." +us,scenario_009,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the spouse's $8,139 employer-sponsored insurance premium from the explicitly listed $60,000 annual gross wages. Using the full wages produces federal AGI of $105,900 and North Carolina taxable income of $54,900." +us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model applied a 4.5% rate to an unsupported approximate $60,800 tax base. The 2026 North Carolina rate is 3.99%, and the traced taxable income is $54,900." +us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model's $97,761 AGI implies that it subtracted the spouse's $8,139 employer-sponsored premium from the listed $60,000 annual wages. The correct AGI is $105,900, producing $54,900 of North Carolina taxable income after the Social Security subtraction and standard deduction." +us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model both reduced wages by the $8,139 employer-sponsored premium and used an unsupported $29,250 North Carolina standard deduction. The correct inputs are $105,900 of federal AGI and a $25,500 joint standard deduction, leaving $54,900 taxable." +us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model's $97,761 federal AGI double-counted the employer-sponsored premium as a reduction from the explicitly listed $60,000 annual gross wages. Federal AGI is $105,900, and the two $25,500 deductions leave $54,900 taxable." +us,scenario_009,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model failed to perform the state taxable-income calculation and replaced the positive liability with zero. Federal AGI of $105,900 less taxable Social Security of $25,500 and the $25,500 standard deduction yields $54,900 taxed at 3.99%." +us,scenario_009,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and credits reduced the tax to zero without identifying any applicable nonrefundable credit. The household has $54,900 of North Carolina taxable income and $2,190.51 of pre-refundable-credit tax." +us,scenario_009,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used the obsolete 4.5% rate rather than North Carolina's 3.99% rate for 2026 and did not derive its tax base. The correct computation is $54,900 × 3.99% = $2,190.51." +us,scenario_009,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used a $21,000 joint standard deduction instead of $25,500. Starting from the correctly adjusted $80,400 North Carolina income, the proper deduction leaves $54,900 rather than $59,400 taxable." +us,scenario_009,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model's $97,761 federal AGI implies an improper $8,139 reduction of the listed $60,000 annual gross wages for employer-sponsored premiums. With $105,900 of federal AGI, North Carolina taxable income is $54,900 and tax is $2,190.51." +us,scenario_009,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. +us,scenario_009,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $25,500 of Social Security included in federal AGI and used an unsupported $26,200 standard deduction. North Carolina taxable income is $105,900 minus $25,500 minus $25,500, or $54,900." +us,scenario_009,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model ultimately derived the correct $54,900 taxable income but applied the obsolete 4.5% rate. North Carolina's 2026 rate is 3.99%, yielding $2,190.51." +us,scenario_009,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model invented a North Carolina earned income tax credit, child credit, and retirement-exclusion credit despite no qualifying child and no such applicable credits in the traced calculation. It also used an unsupported retirement exclusion and obsolete 4.5% rate instead of taxing $54,900 at 3.99%." us,scenario_009,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_009,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_012,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied only CHIP's age and upper-income criteria and omitted the requirement that a CHIP child be ineligible for Medicaid. Child 1 qualifies for Medicaid under the OLDER_CHILD category, which precludes CHIP eligibility." -us,scenario_012,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated income below Mississippi's CHIP ceiling and age under 19 as sufficient conditions. It failed to evaluate the child's Medicaid eligibility under the OLDER_CHILD category, which makes the child ineligible for CHIP." -us,scenario_012,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model stopped after comparing household income with the CHIP limit. It omitted Medicaid precedence: the 10-year-old is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP. -us,scenario_012,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model evaluated CHIP age, income, and absence of employer coverage but never applied the separate-program requirement that the child not qualify for Medicaid. Child 1's OLDER_CHILD Medicaid eligibility precludes CHIP even though income is below the CHIP ceiling." -us,scenario_012,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated falling below the separate CHIP program's income ceiling as sufficient for eligibility. It omitted that Child 1 qualifies for Medicaid under the OLDER_CHILD category, so CHIP eligibility is precluded." -us,scenario_012,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stated the controlling condition—CHIP applies only if the child is not Medicaid-covered—but submitted Yes without testing it. The child is Medicaid-eligible under the OLDER_CHILD category, so that condition fails and CHIP eligibility is No." -us,scenario_012,child1_chip_eligible,minimax-m3,llm_error,categorical_eligibility,False,The model relied solely on age and income relative to the CHIP ceiling. It failed to apply the Medicaid exclusion: OLDER_CHILD Medicaid eligibility takes precedence and bars CHIP eligibility. -us,scenario_012,child1_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model explicitly asserted that the child was not covered by Medicaid instead of evaluating the child's own Medicaid pathway. Child 1 qualifies for Medicaid under the OLDER_CHILD category, and that eligibility precludes CHIP regardless of the CHIP income comparison." -us,scenario_012,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply the Mississippi MAGI Medicaid pathway for an older child and treated the absence of additional child-specific facts as no eligibility support. For a 10-year-old dependent in the OLDER_CHILD category, household MAGI at 0.81 times FPL passes the income test, so the correct eligibility result is yes." -us,scenario_012,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated any low-income minor as WIC categorically eligible. It missed that WIC child eligibility is limited to young children under the program age cutoff, so a 10-year-old child does not qualify through the child pathway even though household income is low." -us,scenario_012,child1_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model used household income below WIC limits as sufficient for a 10-year-old child. It omitted WIC's categorical requirement for the individual applicant, under which this child is outside the eligible child age range and therefore not WIC eligible." +us,scenario_012,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied only CHIP's age and upper-income tests and treated income below 211% FPL as sufficient. It omitted the prerequisite that the child not qualify for Medicaid; this 10-year-old qualifies under Mississippi's OLDER_CHILD Medicaid category, which precludes CHIP." +us,scenario_012,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model concluded that being under age 19 and below the CHIP income ceiling established eligibility. It never tested the child's Medicaid eligibility under the OLDER_CHILD category, whose satisfaction makes the child ineligible for CHIP." +us,scenario_012,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated low income relative to Mississippi's CHIP limit as an affirmative CHIP pathway. It omitted Medicaid precedence: the child qualifies for Medicaid under the OLDER_CHILD category and therefore cannot qualify for CHIP. +us,scenario_012,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model evaluated CHIP age, income, and absence of listed employer coverage but failed to evaluate whether the child qualifies for Medicaid. The child's OLDER_CHILD Medicaid eligibility independently precludes CHIP, so lack of employer-sponsored insurance does not establish CHIP eligibility." +us,scenario_012,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model treated household income below the separate CHIP program's ceiling as sufficient for eligibility. It omitted the Medicaid-ineligibility condition; the child is Medicaid-eligible under the OLDER_CHILD category and is consequently excluded from CHIP. +us,scenario_012,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model correctly stated that CHIP applies only if the child is not Medicaid-covered but then submitted eligibility without performing that condition. The child qualifies for Medicaid under the OLDER_CHILD category, so the model's own stated prerequisite fails and CHIP eligibility is 0." +us,scenario_012,child1_chip_eligible,minimax-m3,llm_error,categorical_eligibility,False,The model used age and income below the CHIP threshold as the complete eligibility test. It skipped the controlling Medicaid screen: OLDER_CHILD Medicaid eligibility applies to this 10-year-old and precludes CHIP. +us,scenario_012,child1_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model explicitly asserted that the child was not covered by Medicaid, incorrectly treating the pregnant spouse's Medicaid status as the relevant inquiry. The child independently qualifies for Medicaid under the OLDER_CHILD category, and that eligibility bars CHIP." +us,scenario_012,child1_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly concluded that household income exceeds the Medicaid threshold for the child and therefore routed the child to CHIP. Under Mississippi's OLDER_CHILD category the 10-year-old is Medicaid-eligible, so Medicaid precedence makes the child ineligible for CHIP." +us,scenario_012,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model failed to place the 10-year-old dependent in Mississippi's OLDER_CHILD Medicaid category and therefore never applied that category's MAGI income test. Applying the test at 0.81 times FPL yields Medicaid eligibility. +us,scenario_012,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated being any minor child as satisfying WIC's age requirement. WIC child eligibility ends at age five, so the 10-year-old is categorically ineligible regardless of the household's low income." +us,scenario_012,child1_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model applied the WIC income limit without first enforcing the program's categorical age restriction. A child age 10 is outside WIC's under-five child category, so passing the income test cannot make the child eligible." +us,scenario_012,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model explicitly misstated WIC's age limit by saying a 10-year-old falls within it. WIC covers children only until age five, making this child categorically ineligible before the income test is considered." us,scenario_012,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_012,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model explicitly derived the correct $4,427 EITC and $1,700 refundable CTC but then submitted $7,093 by adding an unexplained $966. No recovery rebate or other refundable credit applies, so the two stated components total $6,127." -us,scenario_012,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly counted the pregnant spouse or unborn child as an additional federal qualifying child and also omitted the refundable CTC. Only the age-10 child qualifies, producing a $4,427 one-child EITC plus a $1,700 refundable CTC." -us,scenario_012,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used estimated inflation adjustments instead of the 2026 one-child EITC maximum of $4,427, then submitted a value inconsistent even with its own $6,028 component sum. The refundable CTC calculation of $1,700 was correct, yielding $6,127 in total." -us,scenario_012,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model replaced the applicable $4,427 one-child EITC maximum with an unsupported refined estimate of $4,731. At $22,000 the household receives the $4,427 maximum, which combines with the $1,700 refundable CTC to equal $6,127." -us,scenario_012,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used both the wrong EITC amount, $4,367 instead of $4,427, and the wrong refundable CTC cap, $1,800 instead of $1,700. Applying the 2026 amounts gives $4,427 plus $1,700." -us,scenario_012,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model treated the entire $2,000 CTC as refundable, ignoring the 2026 $1,700 per-child refundable cap, and used the prior-year $4,328 EITC maximum. The applicable components are a $4,427 EITC and $1,700 refundable CTC." -us,scenario_012,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model cycled through several incompatible EITC estimates and finally submitted $7,200 despite stating no component calculation that totals that amount. One qualifying child yields the $4,427 maximum EITC and a $1,700 refundable CTC, totaling $6,127." -us,scenario_012,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used $4,355 rather than the 2026 one-child EITC maximum of $4,427 and imposed an obsolete $1,000 refundable CTC cap instead of $1,700. Those corrected components total $6,127." -us,scenario_012,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model understated the refundable CTC by applying a $1,000 cap rather than the 2026 $1,700 cap. Its EITC was also $4 below the applicable $4,427 maximum, so the correct sum is $6,127." -us,scenario_012,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used a $5,571 EITC that is incompatible with the one-qualifying-child schedule and limited the refundable CTC to $1,000. The household instead receives the $4,427 one-child EITC maximum and $1,700 refundable CTC." -us,scenario_012,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used rough values of $4,500 for EITC and $1,000 for refundable CTC. The applicable exact amounts are $4,427 and $1,700, respectively." -us,scenario_012,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied an obsolete $1,000 refundable CTC cap and understated the one-child EITC maximum as $4,320. The 2026 amounts are $1,700 and $4,427." -us,scenario_012,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly applied the $1,700 refundable CTC cap but used the 2025 EITC maximum of $4,328. The 2026 one-child EITC maximum is $4,427, making total refundable credits $6,127." -us,scenario_012,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model reduced the EITC to $2,591 even though $22,000 is below the married-filing-jointly one-child phaseout threshold, and it incorrectly treated zero pre-credit tax as barring the refundable CTC. The household receives the full $4,427 EITC and the $1,700 refundable CTC." -us,scenario_012,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly denied EITC eligibility based on the spouse's age and household structure. A married couple filing jointly with the age-10 qualifying child and $22,000 of earnings qualifies for the $4,427 EITC, in addition to the $1,700 refundable CTC." -us,scenario_012,federal_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The model ignored the instruction to assume tax filing and treated filing status and child eligibility as unspecified. The listed spouses form a joint tax unit, the age-10 child is a qualifying child, and the stated wages support a $4,427 EITC plus a $1,700 refundable CTC." -us,scenario_012,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model overstated both components by using a $4,526 EITC instead of $4,427 and a $1,800 refundable CTC cap instead of $1,700. The applicable sum is $6,127." -us,scenario_012,federal_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly identified the $4,427 one-child EITC but used a $1,800 refundable CTC cap. The 2026 refundable cap is $1,700, producing $6,127." -us,scenario_012,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model applied an EITC phaseout and reduced the credit to $2,479 even though $22,000 is below the married-filing-jointly phaseout threshold. It also omitted the $1,700 refundable CTC; the correct components are $4,427 and $1,700." -us,scenario_012,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an outdated post-sunset regime, setting the refundable CTC cap at $1,000 and the earnings threshold at $3,000, and understated EITC as $4,318. The applicable 2026 amounts are a $4,427 EITC and $1,700 refundable CTC." -us,scenario_012,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model projected the EITC as $4,436 rather than using the applicable $4,427 amount and imposed an obsolete $1,000 refundable CTC cap. The 2026 refundable CTC is capped at $1,700, so total credits equal $6,127." -us,scenario_012,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for federal_refundable_credits, so it failed the required output contract." -us,scenario_012,federal_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly calculated the $1,700 refundable CTC but used $4,401 instead of the 2026 one-child EITC maximum of $4,427. Correcting that parameter raises the total to $6,127." -us,scenario_012,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model set refundable credits to zero despite acknowledging both EITC and refundable CTC, wrongly treating refundable credits as offsetting each other. They add together: $4,427 of EITC plus $1,700 of refundable CTC equals $6,127." -us,scenario_012,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated the pregnancy as a newborn qualifying child, thereby applying the two-child EITC schedule and two refundable CTC caps. An unborn child is not a federal qualifying child here; only the age-10 child counts, yielding a $4,427 EITC and one $1,700 refundable CTC." -us,scenario_012,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated SNAP receipt and free-meal eligibility as unsupported instead of calculating program take-up from the supplied household facts. PolicyEngine calculates about $407.90 in monthly SNAP, which automatically confers categorical eligibility; the household also passes the direct income test at 81% of the poverty guideline." -us,scenario_012,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model applied an ACA Medicaid expansion adult pathway that Mississippi does not provide for this head under PolicyEngine rules. It treated income below 138% FPL as sufficient and skipped the categorical eligibility check that returns medicaid_category NONE. -us,scenario_012,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly rejected Medicaid expansion but incorrectly placed the head into Mississippi's parent/caretaker pathway solely because a 10-year-old child lives in the household. PolicyEngine's categorical screen returns NONE for the head, so the parent/caretaker route is not available and low income does not establish eligibility." -us,scenario_012,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented a pregnancy-related spouse pathway for the head and also misapplied the Mississippi parent/caretaker route. The pregnant spouse's status does not make the 40-year-old head categorically eligible, and PolicyEngine assigns the head medicaid_category NONE despite the child in the household and MAGI of 0.81 FPL." -us,scenario_012,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model reduced Mississippi Medicaid eligibility to an income-threshold test for a low-income adult. PolicyEngine requires a qualifying category before applying income rules, and this head has medicaid_category NONE." -us,scenario_012,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model applied a Mississippi Medicaid expansion rule to the head. PolicyEngine does not provide an expansion adult category here, and the head's medicaid_category is NONE, so income below a MAGI threshold does not confer eligibility." -us,scenario_012,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model assumed Mississippi expanded Medicaid and made the adult eligible based on roughly $22,000 of income. PolicyEngine's categorical eligibility computation finds no qualifying category for the 40-year-old head, so the low MAGI level is insufficient." -us,scenario_012,head_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly treated the head as WIC-eligible because the spouse is pregnant, adding a parent/guardian pathway that is not part of the person-level WIC categorical test. The income test is satisfied for the household, but the head lacks a qualifying WIC category, so household association does not make him eligible." -us,scenario_012,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model misapplied WIC categorical eligibility by allowing a nonpregnant adult household member to qualify through the pregnant spouse. PolicyEngine evaluates WIC eligibility for each person: the spouse qualifies as pregnant, while the 40-year-old head does not meet a WIC category despite household income being below 185% FPL." +us,scenario_012,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model explicitly derived the correct $4,427 EITC and $1,700 refundable CTC, then added an unexplained $966 even though it also stated that no other refundable credit applied. Removing that phantom component yields $6,127." +us,scenario_012,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated the pregnant spouse and unborn fetus as qualifying children and used an inapplicable three-child/Head-of-Household EITC figure. This married household files jointly with one qualifying child, generating a $4,427 EITC and $1,700 refundable CTC." +us,scenario_012,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly identified one qualifying child and the $1,700 refundable CTC but substituted estimated inflation parameters for the enacted 2026 one-child EITC. The applicable EITC is $4,427, not the inconsistent $4,328-to-$4,472 figures implied by its calculation." +us,scenario_012,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model inflated the 2026 one-child maximum EITC to $4,731 after first estimating it near $4,400. At $22,000 of joint earned income the applicable EITC is $4,427, which combines with the $1,700 refundable CTC to produce $6,127." +us,scenario_012,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used both the wrong EITC amount and an $1,800 refundable CTC cap. The 2026 components are a $4,427 one-child EITC and a $1,700 refundable CTC." +us,scenario_012,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly allowed the entire $2,000 CTC to become refundable, ignoring the $1,700 refundable cap, and used 2025 EITC parameters instead of the 2026 $4,427 amount. Zero pre-credit tax does not convert the nonrefundable remainder of the CTC into an additional refundable credit." +us,scenario_012,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model abandoned its own one-child calculations and submitted an unsupported $5,500 EITC solely to force a $7,200 total. The one-child EITC is $4,427 and the refundable CTC is $1,700." +us,scenario_012,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied an obsolete $1,000 refundable CTC cap and $3,000 earnings threshold instead of the 2026 $1,700 cap and $2,500 threshold. It also understated the one-child EITC, which is $4,427." +us,scenario_012,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 refundable CTC rather than the 2026 $1,700 amount. Its EITC was also $4 below the applicable $4,427." +us,scenario_012,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model assigned a $5,571 EITC, exceeding the applicable one-child maximum, and used an obsolete $1,000 refundable CTC. The correct components are $4,427 and $1,700." +us,scenario_012,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used rounded estimates of $4,500 for EITC and an obsolete $1,000 refundable CTC. Applying the 2026 parameters gives $4,427 and $1,700." +us,scenario_012,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 refundable CTC cap and understated the 2026 one-child EITC by $107. The applicable amounts are $1,700 and $4,427." +us,scenario_012,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model carried forward the 2025 one-child maximum EITC of $4,328 rather than applying the 2026 amount of $4,427. Its $1,700 refundable CTC calculation was correct." +us,scenario_012,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model incorrectly reduced the one-child EITC to $2,591 even though $22,000 of joint income remains on the full-credit plateau, and it denied the refundable CTC because pre-credit tax was zero. The refundable CTC exists precisely when its earned-income formula exceeds unused nonrefundable CTC, producing $1,700 here." +us,scenario_012,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly disqualified the household from EITC because the spouse is age 18. A spouse's age does not bar a married couple with a qualifying child from the EITC; this household receives $4,427 of EITC plus $1,700 of refundable CTC." +us,scenario_012,federal_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The model ignored the prompt's filing and take-up assumptions and treated filing status and child eligibility as unspecified. The listed spouses form a joint tax unit, the age-10 child is qualifying, and the $22,000 of wages supports both the $4,427 EITC and $1,700 refundable CTC." +us,scenario_012,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an overstated $4,526 EITC and an $1,800 refundable CTC cap. The 2026 values are $4,427 and $1,700." +us,scenario_012,federal_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly used the $4,427 EITC but applied an $1,800 refundable CTC cap. The applicable refundable CTC is $1,700, reducing its total by $100." +us,scenario_012,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly subjected the one-child joint EITC to a large phaseout at $22,000. Joint income at that level receives the full $4,427 one-child EITC, and the model also omitted the $1,700 refundable CTC." +us,scenario_012,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied obsolete post-sunset rules: a $1,000 refundable CTC cap and $3,000 earned-income threshold. For 2026 the refundable CTC is $1,700, and the one-child EITC is $4,427 rather than $4,318." +us,scenario_012,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model projected the EITC from 2025 instead of using the 2026 $4,427 amount and applied an obsolete $1,000 refundable CTC cap. The refundable CTC is $1,700 because 15% of earnings above $2,500 exceeds that cap." +us,scenario_012,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_refundable_credits, violating the required output contract." +us,scenario_012,federal_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly calculated the $1,700 refundable CTC but used a $4,401 estimated EITC. The applicable 2026 one-child EITC is $4,427, making the total $26 higher." +us,scenario_012,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model set the result to zero despite acknowledging both EITC and refundable CTC and incorrectly described refundable credits as offsetting each other. They add together: $4,427 of EITC plus $1,700 of refundable CTC." +us,scenario_012,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model counted the pregnancy as a newborn qualifying child for both EITC and CTC. An unborn fetus is not a federal qualifying child, so the household has one qualifying child and receives the one-child amounts of $4,427 and $1,700." +us,scenario_012,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model assigned an impossible $6,936 one-child EITC and incorrectly claimed the CTC was fully absorbed by regular tax. Taxable income and pre-credit federal tax are zero, so none of the CTC offsets regular tax; the earned-income formula produces a $1,700 refundable CTC alongside the $4,427 EITC." +us,scenario_012,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated SNAP receipt as unsupported instead of computing the household's approximately $407.90 monthly SNAP benefit and applying SNAP-based categorical eligibility for free school meals. It also omitted the independent income route: the household is at 81% of the federal poverty guideline, below the 130% free-meal threshold." +us,scenario_012,free_school_meals_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model calculated the household's poverty ratio incorrectly: $22,000 places this household at 81% of the applicable federal poverty guideline, not above 130%. It also failed to apply the separate SNAP categorical-eligibility pathway, which independently places the K-12 child in the free tier." +us,scenario_012,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model falsely stated that Mississippi adopted ACA Medicaid expansion and therefore applied the 138% FPL expansion-adult threshold. Mississippi has no expansion-adult pathway, and the head satisfies no other Medicaid category." +us,scenario_012,head_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly rejected Medicaid expansion but treated $22,000, equal to 0.81 FPL in the trace, as low enough for Mississippi's parent/caretaker pathway. That pathway has a far lower income limit, so merely living with a minor child does not place the head in an eligible category at this income." +us,scenario_012,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,The model identified an approximately 27% FPL parent/caretaker limit but then incorrectly asserted that MAGI disregards reduce income at 0.81 FPL below that limit. It also invented pregnancy-derived coverage for the pregnant woman's spouse; pregnancy eligibility applies to the pregnant person and does not create a Medicaid category for the head. +us,scenario_012,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used low household income as sufficient for adult Medicaid eligibility without identifying a covered Mississippi category. The head's 0.81-FPL MAGI does not itself establish eligibility, and the head qualifies through none of the available pathways." +us,scenario_012,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model applied a nonexistent Mississippi Medicaid expansion pathway. Without expansion eligibility, the head must qualify through another covered category, and the head qualifies through none." +us,scenario_012,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model falsely assumed Mississippi expanded Medicaid and treated roughly $22,000 of income as sufficient under an expansion-adult test. Mississippi provides no such pathway, and the head meets no alternative categorical eligibility route." +us,scenario_012,head_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model invented a WIC categorical pathway for a parent or guardian living with a categorically eligible person. The spouse's pregnancy does not make the 40-year-old head eligible; satisfying the household income limit cannot replace the head's missing person-level categorical eligibility. +us,scenario_012,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated the pregnant spouse's categorical eligibility as applying to the entire household. WIC eligibility remains person-specific, and the 40-year-old head falls outside every eligible category even though household income is below the program limit." us,scenario_012,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_012,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_012,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated being below the reduced-price income ceiling as sufficient and failed to apply the mutually exclusive free-meals tier first. The household's 0.81 FPG ratio and categorical eligibility place it in the FREE tier, so reduced-price support is not applicable." -us,scenario_012,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly stated that $22,000 is below 130% FPG and therefore in the free-meal range, but then submitted 1 for reduced-price eligibility, contradicting its own derivation. FREE tier classification excludes positive reduced-price support, so the required value is 0." -us,scenario_012,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model converted the $411 annual energy subsidy into eligibility for an assumed $400 monthly utility allowance and excess-shelter deduction, then also used an erroneous $1,023 maximum allotment. The three-person maximum is $785 and countable income is $1,257.67, yielding about $407.90 per month before the within-year parameter updates." -us,scenario_012,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the 20% earned-income deduction, used incorrect standard-deduction and maximum-allotment values, and treated the energy subsidy as an income adjustment. Applying the earned-income and standard deductions gives $1,257.67 in monthly net income, and $785 minus its 30% contribution is about $407.90 per month." -us,scenario_012,snap,claude-opus-4.7,llm_error,other,False,"The model first derived approximately $4,946 using the correct three-person structure, then replaced that computation with an unsupported $601 monthly figure. Nothing in its stated deductions changes $785 minus roughly 30% of $1,243 into $601." -us,scenario_012,snap,claude-opus-4.8,llm_error,other,False,"The model's stated calculation produces roughly $406 monthly and $4,872 annually, but it discarded that result and submitted an unsupported $514 monthly estimate. No additional deduction or allotment rule in its reasoning generates the submitted $6,168." -us,scenario_012,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model understated the monthly allotment as $378 without specifying parameter values. The traced three-person calculation uses a $785 maximum and $1,257.67 of net income, producing about $407.90 monthly rather than $378." -us,scenario_012,snap,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted the pregnant spouse's unborn child as a fourth SNAP household member; pregnancy does not increase SNAP unit size. It consequently used four-person income limits and a $973 maximum allotment, then submitted $5,136 even though its own repeated calculation stated $7,968." -us,scenario_012,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly treated pregnancy as adding a fourth SNAP household member and therefore used a four-person maximum allotment. The unit contains three people, with a $785 maximum allotment and an expected contribution based on $1,257.67 of monthly net income." -us,scenario_012,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model assumed a $500 monthly standard utility allowance and deducted $316.67 of excess shelter costs, reducing net income to $950. The traced calculation has $1,257.67 of net income for the allotment formula, so the expected contribution is $377.10 rather than $285." -us,scenario_012,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The submitted $482 monthly amount reflects an excessive shelter or utility deduction after invoking a utility allowance triggered by the energy subsidy. The applicable computation retains $1,257.67 of monthly net income and yields about $407.90 per month." -us,scenario_012,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model asserted a $446 monthly allotment without deriving the contribution or identifying the applicable parameters. The three-person maximum of $785 less 30% of $1,257.67 produces about $407.90 monthly, not $446." -us,scenario_012,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied an excess-shelter deduction based on rent and an assumed standard utility allowance, producing an overstated allotment. The traced net-income amount is $1,257.67 monthly, whose 30% contribution reduces the $785 maximum to about $407.90." -us,scenario_012,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's $457 monthly estimate reflects an excessive shelter deduction after the earned-income and standard deductions. The applicable net income is $1,257.67, producing a $377.10 contribution and an allotment of about $407.90." -us,scenario_012,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model invoked an excess-shelter deduction but supplied no figures supporting its $453 monthly allotment. The traced deductions leave $1,257.67 in monthly net income, so the benefit is about $407.90 monthly." -us,scenario_012,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model incorrectly added an unborn child to the SNAP unit and used four-person deductions and a $975 maximum allotment. Pregnancy does not increase SNAP household size, so the applicable unit has three members and a $785 maximum." -us,scenario_012,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave an unsupported $497 monthly estimate despite identifying a three-person household. The applicable $785 maximum less 30% of $1,257.67 yields about $407.90 per month." -us,scenario_012,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model invented a requirement for separate SNAP eligibility flags and defaulted the benefit to zero. The supplied household composition, income, assets, work status, and immigration defaults fully determine eligibility, and the household passes every eligibility test." -us,scenario_012,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used the energy subsidy to trigger a heating-and-cooling allowance and excess-shelter deduction that reduced net income to about $1,000. The traced allotment calculation uses $1,257.67 of monthly net income, producing about $407.90 rather than $483 per month." -us,scenario_012,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The model used a flat $408 monthly estimate and multiplied it by 12. The annual calculation aggregates monthly allotments across changing 2026 parameter periods, producing $4,952.09 rather than a flat-year $4,896." -us,scenario_012,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied heating, cooling, and shelter deductions that increased the monthly allotment to $478. The applicable net income is $1,257.67 and its 30% contribution leaves about $407.90 from the $785 maximum." -us,scenario_012,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model approximated the benefit as exactly $408 for every month and annualized it mechanically. The engine aggregates monthly values across parameter updates during the year, yielding $4,952.09 instead of $4,896." -us,scenario_012,snap,grok-4.3,llm_error,thresholds_rates,False,"The model supplied an unsupported $300 monthly benefit after broadly citing income, housing costs, and the energy subsidy. The three-person formula yields about $407.90 monthly from the $785 maximum and $377.10 expected contribution." -us,scenario_012,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used obsolete or incorrect values of $198 for the standard deduction and $766 for the maximum allotment. The applicable calculation uses net income of $1,257.67 and a $785 maximum, producing about $407.90 monthly before within-year updates." -us,scenario_012,snap,grok-build-0.1,llm_error,period_annualization,False,"The model performed the SNAP deduction and shelter tests on annual totals instead of the program's monthly budget periods, and it treated the annual energy subsidy as a shelter expense. SNAP first calculates monthly net income and allotment, then aggregates those monthly benefits across the year." -us,scenario_012,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no numeric SNAP output or explanation, so the required output contract was not satisfied." -us,scenario_012,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly reached approximately $407.70 monthly but treated that approximation as constant for all 12 months. The annual amount must aggregate monthly allotments across the year's parameter updates, which produces $4,952.09." -us,scenario_012,snap,minimax-m3,llm_error,other,False,"The model submitted zero despite acknowledging that the SNAP amount required computation. The household passes the eligibility tests, and the $785 maximum less the contribution from $1,257.67 of net monthly income produces a positive benefit." -us,scenario_012,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the $411 energy subsidy as unearned income, used incorrect $198 and $768 parameter values, and then submitted $5,111 despite its own arithmetic producing $4,525. The subsidy is not added to SNAP countable income, and the applicable three-person maximum is $785 with $1,257.67 of monthly net income." -us,scenario_012,spouse_chip_eligible,claude-fable-5,llm_error,health_coverage,False,"The model used pregnancy status and income below a pregnancy-coverage threshold as sufficient for CHIP eligibility. It omitted the prior Medicaid screen: the spouse is Medicaid-eligible under the OLDER_CHILD category, which automatically precludes CHIP eligibility." -us,scenario_012,spouse_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model incorrectly treated pregnancy, income, and citizenship-style conditions as the complete CHIP test. It failed to apply the rule excluding anyone already eligible for Medicaid; this spouse qualifies for Medicaid as an OLDER_CHILD." -us,scenario_012,spouse_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,The model conflated CHIP-funded pregnancy coverage with this person's CHIP eligibility output and stopped after finding income below a pregnancy threshold. The spouse's Medicaid eligibility under the OLDER_CHILD category makes `is_chip_eligible` false regardless of that income comparison. -us,scenario_012,spouse_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model treated pregnancy and low household income as establishing CHIP eligibility without checking Medicaid eligibility first. Because the spouse is Medicaid-eligible as an OLDER_CHILD, the CHIP non-Medicaid requirement fails." -us,scenario_012,spouse_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model incorrectly folded pregnancy Medicaid or an unborn-child program into the spouse's personal CHIP eligibility and relied on the 194% FPL comparison. PolicyEngine separately finds the spouse Medicaid-eligible under OLDER_CHILD, and existing Medicaid eligibility bars CHIP." -us,scenario_012,spouse_chip_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model substituted CHIP-funded unborn-child or pregnancy coverage thresholds for the spouse's `is_chip_eligible` test. It omitted that the spouse already qualifies for Medicaid as an OLDER_CHILD, which makes the spouse ineligible for CHIP." -us,scenario_012,spouse_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model applied a pregnant-woman income threshold and conflated Medicaid coverage or CHIP-funded pregnancy programs with personal CHIP eligibility. The spouse is already Medicaid-eligible under the OLDER_CHILD category, so CHIP eligibility is false without regard to the cited FPL calculation." -us,scenario_012,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model missed the Mississippi Medicaid older child pathway for an 18-year-old and treated the absence of an explicitly stated adult Medicaid basis as dispositive. PolicyEngine classifies the spouse as OLDER_CHILD, and with MAGI at 0.67 FPL the spouse satisfies that category's income test, so the correct eligibility output is 1." -us,scenario_012,spouse_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the spouse's listed pregnancy, which is a direct WIC categorical eligibility pathway. It treated the spouse as lacking a qualifying WIC status despite household income falling below the 185% FPL WIC threshold." +us,scenario_012,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the reduced-price ceiling as sufficient for reduced-price eligibility without applying the tier hierarchy. At 81% of the federal poverty guideline, and independently through categorical eligibility, the household qualifies for free meals, which makes reduced-price eligibility false." +us,scenario_012,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly stated that $22,000 falls below the 130% free-meal threshold and that reduced-price support applies only in the 130%–185% band, but then submitted 1 instead of following its own derivation. The FREE tier, also supported by categorical eligibility, excludes positive reduced-price support, so the required output is 0." +us,scenario_012,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model used an erroneous three-person maximum allotment of about $1,023 and then replaced its own resulting $713 monthly calculation with an unsupported $378. The applicable maximum is $785, and applying 30% to $1,257.67 of net income produces about $407.90 per month before later-year parameter updates." +us,scenario_012,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the 20% earned-income deduction, used incorrect standard-deduction and maximum-allotment values, and treated the $411 annual energy subsidy as an income adjustment. The deductions yield $1,257.67 of monthly net income, and the $785 maximum less the $377.10 contribution produces about $407.90 monthly." +us,scenario_012,snap,claude-opus-4.7,llm_error,other,False,"The model first derived approximately the correct $412 monthly result, then discarded that arithmetic and asserted $601 per month without any supporting computation. Using the $785 maximum and 30% of the correctly derived net income keeps the benefit near $408 monthly, not $601." +us,scenario_012,snap,claude-opus-4.8,llm_error,other,False,"The model's stated formula produces about $406 monthly and $4,872 annually, but it replaced that result with an unsupported $514 monthly estimate. No rule in its calculation supplies the extra $108 per month." +us,scenario_012,snap,claude-opus-5,llm_error,thresholds_rates,False,The model reduced the allotment to $378 per month despite the applicable calculation yielding about $407.90 from the $785 maximum and $377.10 household contribution. Its estimate therefore applies an excessive benefit reduction or an understated maximum allotment. +us,scenario_012,snap,claude-sonnet-4.6,llm_error,other,False,"The submitted value of $5,136 contradicts the explanation's repeatedly stated final value of $7,968, violating the requirement that the explanation support and exactly match the numeric output. Independently, the reasoning incorrectly counts the unborn child as a fourth SNAP household member." +us,scenario_012,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model counted the pregnant spouse as two people and therefore used a four-person maximum allotment. An unborn child is not a SNAP household member, so the unit has three people and uses the $785 maximum." +us,scenario_012,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model converted the energy subsidy into a $500 monthly standard utility allowance and deducted $316.67 of excess shelter costs. The traced calculation has $1,257.67 of net income without that shelter deduction, so the contribution is $377.10 rather than $285." +us,scenario_012,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model treated the energy subsidy as triggering a utility allowance that lowered net income and raised the monthly benefit to $482. The applicable net income is $1,257.67, yielding about $407.90 per month rather than $482." +us,scenario_012,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model's $446 monthly estimate exceeds the allotment obtained from the applicable $785 maximum and the $377.10 contribution. Those inputs produce about $407.90 monthly, with annual parameter changes bringing the total to $4,952.09." +us,scenario_012,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied an excess-shelter deduction based on rent plus a standard utility allowance, reducing net income too far. The traced deductions leave $1,257.67 of monthly net income and produce about $407.90 per month, not roughly $497.50." +us,scenario_012,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's shelter deduction lowered net income enough to produce $457 monthly. The applicable deductions leave net income at $1,257.67, so 30% is $377.10 and the allotment is about $407.90." +us,scenario_012,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer implies that the model deducted excess shelter costs and reduced the expected contribution below $377.10. With $1,257.67 of net income and a $785 maximum, the monthly allotment is about $407.90 rather than $453." +us,scenario_012,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model incorrectly added an unborn child to the SNAP unit, used the four-person standard deduction and maximum allotment, and then added a utility allowance to shelter costs. The unit has three members and uses the $785 maximum, producing about $407.90 monthly." +us,scenario_012,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model's roughly $497 monthly estimate is consistent with reducing net income through an unwarranted housing or utility deduction. The applicable net income is $1,257.67, and the $785 maximum less its 30% contribution yields about $407.90 monthly." +us,scenario_012,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model invented a need for additional SNAP eligibility flags even though the prompt supplies the complete household and directs that take-up be assumed. The household passes the gross-income, net-income, asset, work, and immigration tests and therefore receives a positive allotment." +us,scenario_012,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model treated the energy subsidy as triggering a heating-and-cooling allowance and an excess-shelter deduction, lowering net income to about $1,000. The applicable net income is $1,257.67, so the monthly benefit is about $407.90 rather than $483." +us,scenario_012,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The model used a flat $408 monthly estimate for all twelve months. The engine aggregates monthly benefits across the calendar year and incorporates later changes to deductions and poverty-guideline parameters, producing $4,952.09 rather than $4,896." +us,scenario_012,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied rent and heating-or-cooling utility deductions that reduced the expected contribution enough to produce $478 monthly. The traced net income is $1,257.67 without that excess-shelter reduction, yielding about $407.90 monthly." +us,scenario_012,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model correctly approximated the initial monthly benefit as $408 but multiplied that rounded amount uniformly across twelve months. The annual calculation incorporates later-month parameter updates and totals $4,952.09." +us,scenario_012,snap,grok-4.3,llm_error,thresholds_rates,False,"The model's $300 monthly estimate subtracts too much from the applicable maximum allotment. With the $785 three-person maximum and a $377.10 contribution, the monthly benefit is about $407.90." +us,scenario_012,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used outdated values of $198 for the standard deduction and $766 for the maximum allotment. The applicable figures produce $1,257.67 of net income and a $785 maximum, yielding about $407.90 monthly before annual parameter updates." +us,scenario_012,snap,grok-build-0.1,llm_error,period_annualization,False,"The model performed the SNAP shelter comparison and benefit formula on annual totals, even though SNAP computes eligibility and allotments monthly. Monthly calculation uses $1,833.33 of wages, $1,257.67 of net income, and a $785 maximum, then aggregates the monthly allotments." +us,scenario_012,snap,inkling,llm_error,period_annualization,False,"The model rounded the monthly allotment down to $407 and multiplied it uniformly across the year. The unrounded initial calculation is about $407.90, and later-month deduction and guideline updates raise the annual aggregate to $4,952.09." +us,scenario_012,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric SNAP output or explanation, so it failed the required output contract." +us,scenario_012,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly derived essentially the initial monthly calculation but extended one parameter set unchanged across all twelve months. The engine applies later-year updates to the standard deduction and poverty guidelines, so the monthly aggregation totals $4,952.09 rather than $4,892.40." +us,scenario_012,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model returned zero without completing the eligibility or benefit computation. The household passes every SNAP eligibility test, and the $785 maximum less 30% of $1,257.67 in net income produces a positive monthly allotment." +us,scenario_012,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model counted the $411 energy subsidy as unearned SNAP income and used outdated $198 standard-deduction and $768 maximum-allotment figures. The subsidy is not countable income here, and the applicable $785 maximum with $1,257.67 net income yields about $407.90 monthly." +us,scenario_012,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model compared gross annual wages directly with the net-income limit and omitted both the 20% earned-income deduction and the standard deduction. Those deductions reduce monthly net income to $1,257.67, which passes the net test and produces a positive benefit." +us,scenario_012,spouse_chip_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated pregnancy and income below a pregnancy-coverage threshold as sufficient for CHIP. It failed to apply the prior Medicaid-eligibility check: the spouse is Medicaid-eligible under the OLDER_CHILD category, which makes her ineligible for CHIP." +us,scenario_012,spouse_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that pregnancy creates CHIP eligibility regardless of age when income and other conditions are met. It omitted the rule excluding anyone already eligible for Medicaid; the spouse qualifies for Medicaid as an OLDER_CHILD. +us,scenario_012,spouse_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model conflated CHIP-funded pregnancy coverage with the benchmark's individual CHIP eligibility variable. It never applied the controlling exclusion that the spouse's Medicaid eligibility under the OLDER_CHILD category precludes CHIP. +us,scenario_012,spouse_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model used pregnancy and income below pregnancy-related limits as a direct route to CHIP eligibility. It skipped the Medicaid screen, under which the 18-year-old spouse is eligible as an OLDER_CHILD and therefore cannot qualify for CHIP." +us,scenario_012,spouse_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly combined pregnancy Medicaid, CHIP, and an unborn-child program into a single CHIP eligibility pathway for the spouse. PolicyEngine first finds the spouse Medicaid-eligible under the OLDER_CHILD category, and that existing eligibility bars CHIP." +us,scenario_012,spouse_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model substituted CHIP-funded unborn-child or pregnancy coverage thresholds for the spouse's individual CHIP eligibility test. It omitted that her Medicaid eligibility as an OLDER_CHILD precludes CHIP before those CHIP income thresholds can establish eligibility. +us,scenario_012,spouse_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated income below a pregnancy Medicaid or CHIP-funded coverage threshold as establishing CHIP eligibility. It failed to recognize that the spouse already qualifies for Medicaid under the OLDER_CHILD category, which automatically makes her ineligible for CHIP." +us,scenario_012,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model failed to classify the age-18 spouse under Mississippi's OLDER_CHILD Medicaid category and instead treated the supplied facts as establishing no eligibility pathway. Applying that category to MAGI of 0.67 times FPL yields Medicaid eligibility. +us,scenario_012,spouse_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the spouse's explicitly listed pregnancy, which places her in a WIC participant category, and failed to apply the WIC income limit to the household's $22,000 income. Those facts yield eligibility, not 0." us,scenario_012,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_012,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model stopped at a hand-calculated $3,900 taxable-income base and applied a 4% rate instead of carrying the liability through Mississippi’s tax-unit calculation. The unit-level deductions and exemptions eliminate the preliminary individual liability, yielding $0." -us,scenario_012,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model applied a 3.9% rate to its independently constructed $3,900 taxable-income figure and treated $152.10 as the final unit liability. Mississippi’s unit-level calculation reduces the preliminary individual tax to $0 after the applicable deductions and exemptions." -us,scenario_012,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model used an incorrect $6,000 married personal exemption and a 4.4% rate, producing an overstated $11,400 taxable-income base. It also omitted the Mississippi tax-unit step that reduces the household’s preliminary individual liability to $0." -us,scenario_012,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model incorrectly assigned a $2,300 personal exemption to each household member and applied 4% directly to the resulting $10,500 base. The required household output comes from Mississippi’s unit-level calculation, whose deductions and exemptions eliminate the tax liability." -us,scenario_012,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model’s own deduction arithmetic reached $0 taxable income and $0 tax, but it discarded that result and submitted an unsupported $340 based on a hypothetical $6,800 base and 5% rate. The Mississippi tax-unit calculation yields $0, matching the zero result the model computed before its unjustified revision." -us,scenario_013,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented an Arizona aged/disabled Medicaid eligibility pathway for this household and treated disability and age as sufficient to qualify under SSI-related or medically needy rules. PolicyEngine assigns medicaid_category NONE: with MAGI at 1.91 FPL, no SSI receipt, and no applicable Arizona state-specific pathway, the head is not Medicaid eligible." -us,scenario_013,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly applied ALTCS/SSI-related aged or disabled Medicaid logic and then used disregards, medical expense deductions, and a 300% SSI institutional-care threshold to force eligibility. Those pathways do not apply to the listed facts in PolicyEngine: the head receives no SSI, is assigned medicaid_category NONE, and remains above the applicable MAGI Medicaid threshold at 1.91 FPL." -us,scenario_013,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the Arizona standard deduction and applicable exemption treatment that eliminate taxable liability for this elderly single filer, instead applying the 2.5% rate directly to selected income components after a pension subtraction. Its submitted value of 744.10 also contradicts its own stated final calculation of 2.20, so the parsed answer is a computation/submission error on top of the taxable-income error." -us,scenario_013,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Arizona offers no refundable individual income-tax credits. It omitted the refundable increased excise tax credit, which awards this eligible low-income one-person tax unit $25." -us,scenario_013,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model considered credits tied to dependents, rent, and property taxes but omitted Arizona’s increased excise tax credit. That credit requires none of those facts and yields $25 from the tax unit’s $6,736 adjusted gross income and size of one." -us,scenario_013,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated rent, property-tax payments, or dependents as necessary for the increased excise tax credit despite recognizing the household’s low Arizona income. The $6,736 adjusted gross income qualifies the one-person tax unit for the full $25 credit." -us,scenario_013,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly conditioned Arizona’s increased excise tax credit on dependents or positive tax liability. A single tax unit with no dependents can receive the refundable $25 amount at this $6,736 adjusted gross income." -us,scenario_013,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly identified the increased excise tax credit but incorrectly multiplied $25 by separate personal, age, and disability exemptions. PolicyEngine determines the amount from a tax-unit size of one, so the credit is $25 rather than $75." -us,scenario_013,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model omitted Arizona’s refundable increased excise tax credit and focused on unrelated dependent and property-tax pathways. This one-person tax unit qualifies for $25 based on its $6,736 adjusted gross income without those triggers." -us,scenario_013,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,The model incorrectly treated having dependents as necessary for an Arizona refundable credit. The increased excise tax credit gives this qualifying one-person tax unit $25 even with no dependents. -us,scenario_013,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model omitted the Arizona increased excise tax credit. Applying that program to the $6,736 adjusted gross income and tax-unit size of one produces $25." -us,scenario_013,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model incorrectly concluded that no state-specific refundable credit applied. Arizona’s increased excise tax credit applies and contributes $25. -us,scenario_013,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model failed to apply Arizona’s increased excise tax credit. The household’s $6,736 adjusted gross income and one-person tax unit produce a refundable $25 credit." -us,scenario_013,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model omitted the refundable Arizona increased excise tax credit. The qualifying one-person tax unit receives $25 at its $6,736 adjusted gross income." -us,scenario_013,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly concluded that the household exceeded the income limit. The applicable adjusted gross income is $6,736, which qualifies the one-person tax unit for the full $25 increased excise tax credit." -us,scenario_013,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,The model incorrectly stated that Arizona had no refundable credit available to this household. The increased excise tax credit applies and yields $25. -us,scenario_013,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to recognize that the facts trigger Arizona’s increased excise tax credit. A $6,736 adjusted gross income and tax-unit size of one yield $25." -us,scenario_013,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model incorrectly looked for dependents or earned income as the qualifying trigger. Arizona’s increased excise tax credit requires neither here and provides the qualifying one-person tax unit $25. -us,scenario_013,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model omitted Arizona’s increased excise tax credit. The reported facts establish a qualifying $6,736 adjusted gross income for a one-person unit, producing $25." -us,scenario_013,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model incorrectly required property taxes, rent, dependents, or another expense-based trigger for every Arizona refundable credit. The increased excise tax credit depends on the qualifying income and unit size here and equals $25." -us,scenario_013,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model failed to apply the increased excise tax credit to this one-person elderly household. Its $6,736 adjusted gross income qualifies it for $25." -us,scenario_013,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model focused on the Working Poor Tax Credit and income-source requirements while omitting the increased excise tax credit. The latter does not require earnings and provides this qualifying tax unit $25. -us,scenario_013,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model treated property-tax payments or other qualifying expenses as necessary for Arizona refundable credits. The increased excise tax credit requires no such expense and yields $25 based on the household’s qualifying income and unit size. -us,scenario_013,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model omitted the Arizona increased excise tax credit from its assessment. The $6,736 adjusted gross income and tax-unit size of one generate a refundable $25 amount." -us,scenario_013,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,The model incorrectly assumed that refundable Arizona credits require earned income or qualifying property expenses. The increased excise tax credit requires neither and awards this low-income one-person tax unit $25. -us,scenario_013,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $30,544 of gross income as the income tested against the increased excise tax credit limit, thereby including Social Security that does not enter the applicable adjusted gross income. The applicable adjusted gross income is $6,736, which produces the full $25 credit for a one-person tax unit." -us,scenario_013,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,The model incorrectly treated dependents as necessary for an applicable Arizona refundable credit. The increased excise tax credit provides $25 to this qualifying one-person tax unit. -us,scenario_013,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model limited its analysis to earned-income and family-credit pathways requiring earnings or children. It omitted Arizona’s increased excise tax credit, which requires neither and yields $25 from this tax unit’s qualifying income and size." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the separately listed $14,717 employer-sponsored insurance premium from gross wages, even though the supplied gross wage amount is the annual taxable-income input and is not reduced again. Its submitted $5,647 also contradicts its own resulting tax calculation of about $4,545." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $28,000 standard deduction instead of the 2026 married-filing-jointly amount of $32,200 and then incorrectly applied 12% to all taxable income rather than using the graduated brackets. It further submitted an unexplained $6,819 that does not follow from its stated $7,311.36 calculation." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly reached approximately $56,728 of taxable income and approximately $6,324 of bracket tax, but then replaced that derivation with an unsupported $5,235 submission. The correct bracket calculation on $56,727.65 yields $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's reasoning itself derives approximately $6,311 from wages, the married-filing-jointly standard deduction, and the graduated brackets, but it submitted $7,466 instead. That final value is disconnected from every computation stated in its explanation." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived taxable income of about $56,728 and bracket tax of about $6,311, then changed the result to $6,712 without any computation supporting the change. No additional tax or credit adjustment bridges that gap." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a projected $30,000 standard deduction instead of the 2026 married-filing-jointly deduction of $32,200, overstating taxable income by $2,200. It also invented mortgage interest and state-tax estimates despite the instruction that unlisted numeric inputs are zero, although its standard-deduction choice made those estimates nonbinding." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's stated inputs and bracket arithmetic produce about $6,279, but it submitted $8,079 after an unsupported adjustment. The actual $32,200 standard deduction and 2026 brackets produce $6,311.32, with no further adjustment." -us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model double-counted the $14,717 employer-sponsored insurance premium as a reduction from the supplied gross wages. It also applied an assumed TCJA-expiration regime with a smaller standard deduction, personal exemptions, and 15% brackets instead of the operative 2026 $32,200 deduction and 10%/12% brackets." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the supplied gross wages by $14,717 of employer-sponsored insurance premiums. It then substituted an assumed post-TCJA-sunset standard deduction, personal exemptions, and 15% bracket for the operative 2026 married-filing-jointly parameters." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $13,915 is incompatible with applying the 2026 married-filing-jointly standard deduction and graduated rates to $88,927.65 of AGI. Those steps leave $56,727.65 taxable and yield $6,311.32, so the answer reflects an incorrect rate or deduction schedule." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model double-counted the employer-sponsored insurance premium by subtracting $14,717 from the supplied gross wages. It also used assumed TCJA-expiration deductions, personal exemptions, and pre-TCJA brackets rather than the operative 2026 $32,200 standard deduction and married-filing-jointly brackets." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced gross wages by the separately listed $14,717 employer-sponsored insurance premium. It compounded that error by applying an assumed TCJA-expiration $16,000 deduction and a 15% bracket instead of the operative $32,200 standard deduction and 10%/12% schedule." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted $14,717 of employer-sponsored insurance premiums from the supplied gross wages and then invoked personal exemptions. The correct computation begins with $88,927.65 of AGI and subtracts only the operative $32,200 standard deduction." -us,scenario_014,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $30,000 married-filing-jointly standard deduction instead of $32,200, overstating taxable income by $2,200. Applying the operative deduction and 2026 brackets yields $6,311.32 rather than $6,591.36." -us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The submitted $10,458 does not result from the stated married-filing-jointly standard-deduction calculation. With $56,727.65 of taxable income, the 2026 graduated brackets yield $6,311.32, so the answer applied the wrong effective rate or bracket schedule." -us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model invented nonrefundable credits that fully offset the liability even though no qualifying credit facts are listed. The standard deduction leaves $56,727.65 taxable, and no nonrefundable credit reduces the resulting $6,311.32 tax." -us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model applied a supposed post-TCJA schedule with personal exemptions and arrived at $61,828 of taxable income instead of using the operative $32,200 standard deduction. Correct taxable income is $56,727.65, taxed under the 2026 married-filing-jointly brackets for $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $88,927.65 of wages as fully eliminated by deductions and exemptions. The $32,200 standard deduction leaves $56,727.65 of taxable income, while child support is not deductible and supplies no offset." -us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model substituted an assumed post-TCJA-sunset regime with a $17,018 standard deduction, personal exemptions, and a 15% bracket. The operative 2026 calculation instead uses the $32,200 married-filing-jointly standard deduction and 10%/12% brackets, producing $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted the $14,717 employer-sponsored insurance premium by subtracting it from the supplied gross wages. Starting from the full $88,927.65 of wage AGI and subtracting the $32,200 standard deduction leaves $56,727.65 taxable, not $43,371." +us,scenario_012,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model stopped after applying a 4% rate to its reconstructed $3,900 taxable-income figure. It omitted the Mississippi unit-level computation that reduces the primary earner's $118 individual liability to $0 before state liabilities are aggregated." +us,scenario_012,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model treated $22,000 less a $4,600 joint standard deduction and $13,500 of exemptions as the final Mississippi tax base and directly applied 3.9%. It failed to carry the calculation through Mississippi's unit-level deductions and exemptions, which reduce the household tax before credits to $0." +us,scenario_012,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model used an incorrect $6,000 married personal exemption and applied a 4.4% rate to $11,400. More fundamentally, it bypassed the Mississippi unit-level tax calculation that reduces the individual liability to a $0 household liability." +us,scenario_012,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model incorrectly assigned a $2,300 personal exemption to each household member and treated $10,500 as the final taxable base. It also omitted the Mississippi unit-level deductions and exemptions that eliminate the household liability after the individual calculation." +us,scenario_012,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model's own deduction-and-exemption calculation repeatedly reached $0 tax, but it discarded that result and submitted an unsupported $340 estimate based on an invented $6,800 taxable base and 5% rate. The applicable Mississippi unit-level calculation yields $0." +us,scenario_012,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model used an inapplicable bracket calculation—$1,000 as 5% of the first $10,000 is also arithmetically wrong—and invented a $970 family credit from unsupported component amounts. It failed to apply Mississippi's actual unit-level deductions and exemptions, which reduce tax before refundable credits to $0." +us,scenario_013,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model converted age and disability into eligibility without identifying a satisfied Arizona Medicaid category. It invented qualification under medically needy or SSI-related aged/disabled rules even though SSI is zero and the person qualifies through none of the available pathways. +us,scenario_013,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model improperly applied the ALTCS 300%-of-SSI income ceiling without the required long-term-care eligibility facts, which are unlisted and therefore false, and it disregarded the stated $58,700 of assets despite citing a $2,000 limit. It also invented a large Social Security exclusion and medical-expense spend-down that do not place this person into an Arizona aged/disabled Medicaid category." +us,scenario_013,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted Arizona’s standard deduction, which eliminates taxable income and yields $0 before refundable credits. Its submitted $744.10 also directly contradicts its own stated calculation of $2.20, so the final value does not follow from its reasoning." +us,scenario_013,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Arizona offers no refundable individual income-tax credits. It omitted the refundable increased excise tax credit, for which the $6,736 adjusted gross income and tax-unit size of one produce $25." +us,scenario_013,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model limited Arizona refundable credits to pathways requiring dependents, rent, or property-tax payments. The increased excise tax credit requires none of those facts here and awards this eligible one-person tax unit $25." +us,scenario_013,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model wrongly grouped the increased excise tax credit with credits requiring property-tax or rent payments and then declared its conditions unmet. The household's $6,736 adjusted gross income satisfies the increased-excise-credit income test, yielding $25 without dependents or housing expenses." +us,scenario_013,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,The model incorrectly treated the absence of dependents and state tax liability as disqualifying. Arizona's increased excise tax credit is refundable and grants $25 to this eligible low-income one-person tax unit regardless of positive tax liability. +us,scenario_013,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly identified the increased excise tax credit and its $25 base amount, then incorrectly counted age and disability as two additional credit-bearing exemptions. Tax-unit size is one for this computation, so the full credit is $25, not three times $25." +us,scenario_013,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model omitted Arizona's refundable increased excise tax credit and focused on family and property-tax pathways. This one-person unit qualifies through its $6,736 adjusted gross income and receives $25 without dependents or property-tax facts." +us,scenario_013,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,The model treated having no dependents as eliminating all Arizona refundable credits. The increased excise tax credit covers an eligible taxpayer without dependents and provides $25 for this one-person unit. +us,scenario_013,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The zero answer omits the Arizona increased excise tax credit. Applying its income eligibility rule to adjusted gross income of $6,736 and a tax-unit size of one yields $25." +us,scenario_013,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model's claim that there are no state-specific refundable credits overlooks Arizona's increased excise tax credit. This eligible one-person tax unit receives the full $25 amount. +us,scenario_013,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model failed to apply the Arizona increased excise tax credit's low-income eligibility pathway. The household's $6,736 adjusted gross income qualifies it for $25." +us,scenario_013,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model omitted the refundable Arizona increased excise tax credit. Its income test and one-person amount produce a $25 credit for this household. +us,scenario_013,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly concluded that household income exceeds Arizona's refundable-credit limit. The increased excise tax credit uses the applicable adjusted gross income, which is $6,736 here, and that qualifying amount yields a $25 credit." +us,scenario_013,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly stated that Arizona has no refundable credit available to this household. The increased excise tax credit applies based on the unit's qualifying $6,736 adjusted gross income and equals $25." +us,scenario_013,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model failed to trigger Arizona's increased excise tax credit. The household meets its income eligibility conditions and receives $25 for a tax-unit size of one. +us,scenario_013,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly looked for dependents or earned income as the necessary eligibility trigger. Arizona's increased excise tax credit requires neither here; the qualifying $6,736 adjusted gross income produces $25." +us,scenario_013,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model omitted the applicable Arizona increased excise tax credit. This one-person household qualifies under its adjusted-gross-income test and receives $25. +us,scenario_013,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model wrongly required property tax, rent, dependents, or another expense-based trigger for every Arizona refundable credit. The increased excise tax credit instead applies through the household's qualifying $6,736 adjusted gross income and equals $25." +us,scenario_013,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model failed to recognize that a one-person elderly household can qualify for Arizona's increased excise tax credit. Its qualifying adjusted gross income produces the full $25 amount. +us,scenario_013,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model considered a working-poor credit and treated the household's income sources as disqualifying, but omitted the increased excise tax credit. That credit does not require earnings and awards $25 based on this unit's qualifying adjusted gross income." +us,scenario_013,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model treated missing property-tax expenses as eliminating Arizona refundable credits. The increased excise tax credit is not conditioned on property-tax payments, and this income-eligible unit receives $25." +us,scenario_013,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model's blanket conclusion omitted Arizona's increased excise tax credit. Applying that credit to the $6,736 adjusted gross income and one-person tax unit yields $25." +us,scenario_013,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model incorrectly treated zero taxable income, no dependents, and no listed take-up as disqualifying. The increased excise tax credit is refundable, does not require dependents, and is claimed under the prompt's filing and take-up assumption, producing $25." +us,scenario_013,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,The model restricted Arizona refundable credits to earned-income or expense-based programs. The increased excise tax credit requires no earned income or qualifying expense here and provides $25 based on the household's qualifying adjusted gross income. +us,scenario_013,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model compared total cash receipts of $30,544 with the increased-excise-credit limit instead of using the applicable Arizona adjusted gross income. Social Security treatment reduces adjusted gross income to $6,736 for this test, so the household qualifies for $25." +us,scenario_013,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,The model incorrectly treated the absence of dependents as eliminating all qualifying Arizona refundable credits. The increased excise tax credit applies to this eligible one-person tax unit and equals $25. +us,scenario_013,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model focused on earned-income and family-credit pathways and wrongly required children or earnings. Arizona's increased excise tax credit requires neither for this household; its qualifying adjusted gross income yields $25. +us,scenario_013,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model incorrectly used the absence of state tax liability to support a zero refundable-credit result. Arizona's increased excise tax credit is refundable and provides this income-eligible one-person unit $25 even with no liability to offset. +us,scenario_014,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the separately listed $14,717 of employer-sponsored insurance premiums from gross wages, even though the benchmark's employment-income input remains $88,927.65 for AGI. Its own resulting tax calculation was $4,545.32, so the submitted $5,647 also contradicts its stated derivation." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $28,000 standard deduction instead of the 2026 married-filing-jointly amount of $32,200 and then applied 12% to all taxable income instead of using the 10% bracket first. Its unexplained adjustment from $7,311.36 to $6,819 did not implement the correct bracket calculation." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly identified taxable income near $56,728 and computed tax near $6,324 using its estimated bracket, but then submitted $5,235 without any supporting computation. Applying the $24,800 threshold to $56,727.65 yields $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's reasoning reached the correct bracket-based result of about $6,311, but it submitted $7,466 instead. The submitted value is disconnected from its stated taxable-income and bracket calculation." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model explicitly calculated about $6,311 from taxable income near $56,728 and the correct $24,800 first-bracket ceiling, then replaced that result with $6,712. No stated tax rule or computation supports the submitted value." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $30,000 standard deduction instead of $32,200, leaving taxable income $2,200 too high. It also reverted to the 2025 $23,850 bracket threshold despite discussing an inflation adjustment, rather than applying the 2026 $24,800 threshold." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's stated inputs and brackets produced about $6,279, close to the correct bracket computation, but it submitted $8,079 after an unsupported adjustment. No nonrefundable credit or additional tax in its reasoning explains that increase." +us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model double-counted the $14,717 employer-sponsored insurance premium as a deduction from the listed wages. It also incorrectly applied a post-sunset regime with a $15,400 standard deduction, personal exemptions, and 15% bracket instead of the operative 2026 $32,200 deduction and 10%/12% brackets." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model separately deducted the $14,717 employer-sponsored insurance premium from employment income and then applied an expired-law structure with a reduced standard deduction, personal exemptions, and a 15% bracket. The correct computation starts with $88,927.65 of AGI and uses the $32,200 standard deduction under the operative 2026 rules." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $13,915 cannot result from applying the 2026 married-filing-jointly brackets to $56,727.65 of taxable income. The required 10% tax through $24,800 plus 12% on the remainder equals $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced wages by the separately listed $14,717 insurance premium and used a post-sunset combination of deductions, exemptions, and pre-TCJA brackets. Operative 2026 law instead leaves $88,927.65 in AGI and subtracts the $32,200 standard deduction." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model double-counted the employer-sponsored insurance premium as a reduction of the listed wages. It then used a $16,000 standard deduction and a 15% bracket based on an inapplicable TCJA-expiration assumption rather than the 2026 $32,200 deduction and 10%/12% schedule." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the spouse's listed wages by $14,717 and invoked personal exemptions. The correct AGI is $88,927.65, followed by only the $32,200 standard deduction before applying the 2026 brackets." +us,scenario_014,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $30,000 married-filing-jointly standard deduction instead of $32,200, overstating taxable income by $2,200. It also used a $24,000 first-bracket ceiling instead of $24,800." +us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The submitted $10,458 is inconsistent with the stated use of the standard deduction and the applicable married-filing-jointly brackets. Taxable income is $56,727.65, and applying 10% through $24,800 plus 12% above that yields $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model invented nonrefundable credits that fully offset the tax even though the household has no listed dependent or qualifying credit facts. The $32,200 standard deduction leaves $56,727.65 taxable, producing $6,311.32 before refundable credits." +us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model applied an inapplicable post-TCJA regime with personal exemptions and derived taxable income of about $61,828. The operative computation uses the $32,200 standard deduction without personal exemptions, leaving $56,727.65 taxable." +us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $88,927.65 of wages as fully eliminated by deductions and exemptions. The standard deduction removes only $32,200, leaving $56,727.65 subject to federal income tax." +us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model applied an inapplicable post-sunset structure with a $17,018 standard deduction, personal exemptions, and a 15% bracket. The operative 2026 calculation uses a $32,200 standard deduction, no personal exemptions, and the 10%/12% brackets at this income." +us,scenario_014,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the separately listed $14,717 employer-sponsored insurance premium from the wage input, reducing AGI to $74,211. The correct AGI remains $88,927.65, and the $32,200 standard deduction leaves $56,727.65 taxable." +us,scenario_014,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an inapplicable pre-TCJA structure with a reduced standard deduction, two personal exemptions, and a 15% bracket. The operative 2026 rules use the $32,200 married-filing-jointly standard deduction and 10%/12% brackets for this taxable income." us,scenario_014,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_014,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the standard deduction and a personal exemption as reducing $88,927.65 of wages to near zero. The operative $32,200 standard deduction leaves $56,727.65 taxable, and no personal exemption eliminates that income." -us,scenario_014,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model taxed $18,480 of veterans benefits while excluding the head's veterans benefits, even though both veterans-benefit amounts are nontaxable, and it invented two children and a $4,000 CTC despite no children being listed. Its negative $17,600 submission also follows from neither its stated $8,616.80 regular tax nor its stated $4,616.80 post-credit tax." -us,scenario_014,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_014,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model correctly used the full wage base and the two federal FICA rates, but then invented a $6.21 West Virginia employee payroll tax at an estimated 0.2% rate. West Virginia imposes no such mandatory employee-side payroll tax here, and the engine’s component calculations total $6,802.96." -us,scenario_014,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted $14,717 of employer-sponsored insurance premiums from FICA wages. The prompt does not identify those premiums as pre-tax cafeteria-plan deductions, so Social Security and Medicare taxes apply to the full $88,928 of wages and total $6,802.96." -us,scenario_014,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the spouse’s employer-sponsored insurance premiums as automatically exempt from FICA and reduced taxable wages to $74,211. With no stated pre-tax treatment, the FICA base remains $88,928, producing $5,513.51 of Social Security tax plus $1,289.45 of Medicare tax." -us,scenario_014,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $14,717 insurance premium from FICA wages merely because it was employer-sponsored. The full $88,928 wage amount is subject to employee Social Security and Medicare taxes, yielding $6,802.96." -us,scenario_014,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model misclassified the listed employer-sponsored insurance premium as a pre-tax FICA exclusion and used a $74,211 payroll-tax base. The specified gross wages remain the FICA base, so the two employee federal payroll-tax components total $6,802.96." -us,scenario_014,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model subtracted the $14,717 employer-sponsored insurance premium without any fact establishing pre-tax cafeteria-plan treatment. Applying 6.2% Social Security and 1.45% Medicare to the engine’s full-wage base produces $6,802.96." -us,scenario_014,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The submitted $13,591.63 contradicts the model’s own stated computation: employee FICA at 7.65% on $88,928 is about $6,803, not $13,592. Its answer effectively doubles the employee-side liability despite explicitly excluding employer payroll taxes." -us,scenario_014,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The stated 7.65% employee FICA rate on the only listed wages of $88,928 yields $6,802.96 under the engine’s component calculations, not $12,450. The submitted amount therefore uses an inflated payroll-tax base or includes taxes outside the requested employee-side scope." -us,scenario_014,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages by the $14,717 employer-sponsored insurance premium. Because no pre-tax payroll exclusion is specified, the full $88,928 is subject to employee Social Security and Medicare taxes, totaling $6,802.96." -us,scenario_014,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required output contract." -us,scenario_014,payroll_tax,minimax-m3,llm_error,other,False,"The model’s own component arithmetic produces approximately $6,803, but it submitted $6,581. The engine’s precise Social Security and Medicare components are $5,513.51 and $1,289.45, which sum to $6,802.96." -us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reached approximately $84,928 of taxable income but applied an incorrect projected schedule of 2.22%, 2.96%, 3.33%, 4.44%, and 4.82%. It then abandoned even its own $3,256 bracket calculation and submitted $3,374 without a valid adjustment." -us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the spouse's $14,717 employer-sponsored insurance premium from the reported $88,928 of gross wages and also treated child support as deductible. West Virginia taxable income is $84,927.65 after the $4,000 personal exemption, not approximately $74,411, and an estimated average rate cannot replace the statutory bracket calculation." -us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model computed about $3,255 under its chosen brackets, then reduced the result to $2,877 for unspecified nonrefundable credits despite explicitly finding that the family tax credit was unavailable. No applicable credit supports that $378 reduction, and the correct 2026 schedule yields $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used approximate and conflicting West Virginia rates, first calculating roughly $3,130 and then imposing an unsupported further rate-cut adjustment to reach $2,849. The applicable 2026 joint-filer schedule on $84,927.65 produces $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model correctly identified roughly $84,928 of taxable income but asserted that tax fell to $1,655 after nonrefundable credits without naming an eligible credit. No such credit applies, and the statutory bracket computation produces $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model repeatedly calculated tax near $3,360 under an approximate schedule, correctly concluded that no medical deduction or identified credit applied, and then nevertheless reduced liability to $1,919 for invented additional rate cuts or a low-income credit. This income is not entitled to the asserted low-income reduction; the applicable schedule yields $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model reached the correct taxable-income base but used a broad estimated rate range instead of the applicable 2026 West Virginia bracket parameters. That approximation produced $3,200 rather than the bracket result of $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model invented a $15,400 West Virginia standard deduction and began from an understated $74,211 AGI. West Virginia provides no such standard deduction here; AGI is $88,927.65 and only the $4,000 personal exemption reduces it to $84,927.65." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used $74,211 as federal AGI, reflecting an improper subtraction of the separately listed employer-sponsored premium from gross wages. The trace uses AGI of $88,927.65 and taxable income of $84,927.65, so its $70,211 tax base and resulting tax are understated." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $4,125 estimate is inconsistent with the required computation: $88,927.65 of AGI less $4,000 of personal exemptions, followed by the 2026 West Virginia joint-filer brackets. That derivation yields $3,092.19, so the answer applied an excessive effective rate or failed to perform the bracket calculation." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $74,211 and therefore used taxable income of $70,211. The correct base is $88,927.65 less $4,000 of personal exemptions, or $84,927.65, before applying the state brackets." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly used $74,211 of federal AGI, producing a $70,211 West Virginia tax base after exemptions. The engine uses $88,927.65 of AGI and $84,927.65 of taxable income, which yields $3,092.19 under the applicable brackets." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model based its calculation on $70,211 of taxable income after starting from an erroneous $74,211 AGI. West Virginia taxable income is $84,927.65 because the reported wages enter AGI without another subtraction for the listed employer premium." -us,scenario_014,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented an $8,000 West Virginia standard deduction in addition to the $4,000 personal exemption. It also multiplied the entire remaining base by the top marginal rate instead of applying each graduated bracket; only the personal exemptions reduce AGI here." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model supplied no calculation supporting $2,331. Applying the 2026 West Virginia joint-filer brackets to $84,927.65 of taxable income yields $3,092.19, so its answer embodies an understated effective rate." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model invented deductions or nonrefundable items that fully eliminated the liability. The household has $84,927.65 of West Virginia taxable income and no applicable offset that reduces the bracket tax of $3,092.19 to zero." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly formed approximately $84,928 of taxable income but applied the wrong 2026 rate schedule, including a 4.82% top rate. The applicable West Virginia joint-filer parameters produce $3,092.19 rather than $3,255.03." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used the correct general tax base but an incorrect West Virginia rate schedule. Applying the applicable 2026 joint-filer brackets to $84,927.65 yields $3,092.19, not $3,217.53." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly subtracted the two personal exemptions but applied a projected rate schedule that overstates the bracket tax. The applicable 2026 schedule produces $3,092.19 on $84,927.65." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The answer implies the same overstated rate schedule used by the nearby $3,255.03 calculations. With no nonrefundable credits, the correct 2026 bracket computation on $84,927.65 is $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions reduce West Virginia tax to zero without identifying any deduction capable of eliminating $84,927.65 of taxable income. Only $4,000 of personal exemptions applies against $88,927.65 of AGI, leaving positive bracket tax of $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a 2024-style schedule, including a 5.12% top rate, as the 2026 schedule. Those obsolete rates overstate the tax; the applicable 2026 parameters yield $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model imported the federal standard deduction into the West Virginia AGI-based calculation and also began from the understated AGI of $74,211. West Virginia does not subtract that federal standard deduction here; taxable income is $84,927.65 after the state personal exemptions." -us,scenario_014,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model double-counted the $14,717 employer-sponsored premium as a wage deduction, invented an $8,000 West Virginia standard deduction, and then used a nonexistent 4.5% flat tax. The correct calculation starts from $88,927.65 of AGI, subtracts $4,000 of personal exemptions, and applies graduated brackets." -us,scenario_014,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly reached approximately $84,928 of taxable income but used an obsolete 5.12% marginal rate above $60,000 and an incorrect $2,183.50 bracket base. The applicable 2026 schedule yields $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the personal exemptions as large enough to erase all taxable wages. The two exemptions total only $4,000, leaving $84,927.65 of West Virginia taxable income and $3,092.19 of tax." -us,scenario_014,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model first double-counted the $14,717 employer-sponsored premium as a deduction from wages, then reversed West Virginia's progressive rate order, and finally invented a 4% family tax credit. The correct base is $84,927.65, and the applicable graduated schedule with no such credit yields $3,092.19." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own stated taxable income and bracket calculation produce approximately $2,969, but it submitted $3,103 without any corresponding tax component. It replaced its completed arithmetic with an unsupported final number." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model used an obsolete $14,600 standard deduction and incorrectly included workers' compensation in gross income, even though workers' compensation is excluded. Its own calculation then produced $3,094.28, yet it submitted $3,509 without identifying any additional tax." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly reached taxable income near $26,809 and calculated tax near $2,974, then submitted $3,268 without adding any valid tax component. The submitted value contradicts its own derivation." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model treated the listed $45,000 gross wages as already net of the traditional 401(k) contribution and therefore failed to subtract the deductible contribution. That overstated AGI and taxable income by roughly $1,042." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model incorrectly described the $4,510 capital loss as fully deductible despite the $3,000 annual ordinary-income offset limit, omitted the traditional 401(k) deduction, and used the wrong standard deduction. Its stated computation yielded about $2,841, but it submitted $4,172 with no supporting calculation." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model repeatedly substituted 2025 and speculative projected parameters for the applicable 2026 $16,100 standard deduction and rate schedule. Its final $3,111 specifically uses the 2025 $15,000 standard deduction and $11,925 first-bracket threshold instead of the 2026 parameters." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model incorrectly awarded a Saver's Credit at AGI of about $42,909 and then asserted further unidentified reductions. It also submitted $1,750 even though its own tax-minus-credit calculation was $2,876 and no child or disability credit supplied the difference." -us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted the separately listed $2,080 health-insurance premium from wages as though it were a pre-tax payroll exclusion, despite the prompt not establishing that treatment. It also used obsolete $15,000 and $11,600 deduction-and-bracket parameters instead of the 2026 schedule." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset and applied a pre-TCJA personal exemption, reduced standard deduction, and 15% second bracket. The applicable 2026 computation instead uses the $16,100 standard deduction and the 10%/12% rate schedule." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied only an approximate AGI and no deduction, bracket, or credit calculation capable of producing $1,541. Taxable income of $26,809.38 under the 2026 single schedule yields $2,969.13, and there are no nonrefundable credits reducing it to the submitted amount." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $2,080 health-insurance premium from wages and assumed a TCJA sunset with a personal exemption. Those choices replaced the applicable $16,100 standard-deduction computation and distorted both taxable income and the rate schedule." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly derived AGI of about $42,909 but used a $15,000 standard deduction and an $11,600 first-bracket threshold. Using the applicable $16,100 deduction and 2026 rate schedule lowers taxable income to $26,809.38 and tax to $2,969.13." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly treated the separately listed $2,080 health premium as a pre-tax wage reduction. It also used a $15,700 standard deduction rather than $16,100, understating taxable income and tax." -us,scenario_015,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly derived AGI near $42,909 but substituted an estimated $15,400 standard deduction and $12,247 bracket threshold. The applicable $16,100 standard deduction and 2026 schedule produce $2,969.13." -us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model assumed the standard deduction and unspecified nonrefundable credits eliminated all liability. The standard deduction leaves $26,809.38 taxable, and no nonrefundable credit is available, so regular tax remains $2,969.13." -us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated child support as deductible even though personal child-support payments do not reduce federal taxable income. It supplied no arithmetic explaining $5,199; the actual deductions leave $26,809.38 taxable and no nonrefundable credits apply." -us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model stated the correct taxable-income inputs and a bracket calculation of 10% on $12,400 plus 12% on $14,409, which equals about $2,969 rather than $3,211.08. The submitted value is an arithmetic or transcription error unsupported by its reasoning." -us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model netted all $1,351 of desired traditional and Roth retirement contributions against wages, although Roth contributions are not deductible and the deductible traditional amounts total only about $1,091. It also used the full $4,510 capital loss rather than the $3,000 ordinary-income deduction limit." -us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed reversion to pre-TCJA rules, adding a personal exemption and applying a 15% second bracket. The applicable 2026 rules use a $16,100 standard deduction, no personal exemption in this computation, and a 12% second bracket." -us,scenario_015,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model imposed a post-TCJA-sunset regime with an inflation-adjusted personal exemption and a 15% second bracket. The applicable 2026 computation uses the $16,100 standard deduction and the 10%/12% schedule, producing $2,969.13." -us,scenario_015,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for the requested variable. -us,scenario_015,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model assumed disability and unspecified credits reduced the tax to zero. Disability alone supplies no credit here, no other nonrefundable credits apply, and the $16,100 standard deduction still leaves $26,809.38 taxable." -us,scenario_015,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model incorrectly included $960 of workers' compensation in gross income and used a $15,750 standard deduction rather than $16,100. Even with those errors its own bracket calculation was $3,137.16, so the submitted $2,104 is an unsupported final-number substitution." -us,scenario_015,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_015,head_medicaid_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model invented a disability-based Indiana Medicaid eligibility route for this case and treated the head's disability and part-time work as enough to qualify through ABD or a working-disabled buy-in pathway. PolicyEngine's trace places the person in no Medicaid category, with MAGI at 2.69 times FPL and SSI received equal to zero, so neither a MAGI category nor an SSI-linked disabled pathway establishes eligibility." -us,scenario_015,head_medicaid_eligible,minimax-m3,llm_error,health_coverage,False,"The model collapsed Indiana Medicaid expansion and disability status into automatic Medicaid eligibility. The head is age 36 with MAGI at 2.69 times FPL and no qualifying Medicaid category, so expansion coverage and disability status do not overcome the income/category failure." -us,scenario_015,head_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model replaced Medicare's under-65 entitlement requirements with a simple age-65-or-disabled rule. The head's disability flag does not establish SSDI-based Medicare entitlement or end-stage renal disease, so the age-36 head is not eligible." -us,scenario_015,head_medicare_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model treated reported disability as independently sufficient for Medicare eligibility. It failed to require an under-65 Medicare entitlement pathway, such as the applicable SSDI entitlement period or end-stage renal disease, neither of which is present." -us,scenario_015,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly subtracted the $2,080 health-insurance premium from FICA wages. Applying 7.65% to the full $45,000 wage base yields $3,442.50, not $3,283." -us,scenario_015,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model treated the separately listed $2,080 health-insurance premium as a pre-tax cafeteria-plan deduction and reduced FICA wages to $42,920. The payroll-tax base remains $45,000, yielding $2,790 of Social Security tax and $652.50 of Medicare tax." -us,scenario_015,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model wrongly deducted the $2,080 health-insurance premium from wages subject to Social Security and Medicare taxes. FICA applies to all $45,000 of wages, so the total is $3,442.50." -us,scenario_015,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model derived the correct components—$2,790 of Social Security tax and $652.50 of Medicare tax—but added or transcribed them incorrectly in its submitted value. Those components total $3,442.50, not $3,445.50." -us,scenario_015,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's stated Social Security and rounded Medicare components do not add to its answer: $2,790 plus $653 equals $3,443, not $3,419. Using the exact Medicare amount of $652.50 produces $3,442.50." -us,scenario_015,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll-tax output or explanation. It therefore failed the required structured-output contract rather than completing the payroll-tax calculation. -us,scenario_015,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model correctly calculated $3,442.50 of federal employee FICA, then replaced it with $3,495 by inventing an Indiana county-tax adjustment. County income taxes do not belong in this employee payroll-tax output, and no mandatory Indiana employee payroll tax applies." -us,scenario_015,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages by the $1,042 traditional 401(k) contribution; elective traditional 401(k) deferrals remain subject to Social Security and Medicare taxes. It also submitted $3,425.67 despite its own scratch calculation of $3,362.79; applying FICA to the full $45,000 yields $3,442.50." -us,scenario_015,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived the correct $41,909 taxable-income base and an approximately $1,236 liability, then arbitrarily submitted $1,234 as “minor rounding.” Applying the rate to the unrounded $41,909.38 base yields $1,236.33, so ordinary rounding cannot produce its answer." -us,scenario_015,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a 3.15% Indiana rate instead of the applicable 2026 rate and incorrectly included workers’ compensation in gross income before later inventing unspecified disability exemptions and credits. Indiana starts from $42,909.38 of AGI, subtracts only the $1,000 base exemption, and taxes $41,909.38." -us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model improperly claimed an additional $500 disability exemption and used the wrong rate, then submitted $1,352 even though every calculation in its explanation produced $1,263 or $1,304. The correct computation uses only the $1,000 base exemption and yields $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model omitted the $1,042 traditional 401(k) reduction from AGI, then submitted $1,392 despite computing about $1,267 from its own stated base and rate. The 401(k) reduction produces AGI of $42,909.38, and the $1,000 exemption leaves $41,909.38 to tax." -us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an incorrect 2.85% Indiana rate and an unsupported AGI of about $42,441. The trace establishes AGI of $42,909.38 and taxable income of $41,909.38, producing $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted a nonexistent $1,000 disability exemption and an unsupported $2,080 Indiana health-insurance-premium deduction. Only the $1,000 base exemption reduces the $42,909.38 AGI, leaving $41,909.38 rather than $38,829." -us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model incorrectly added a $1,000 disability exemption and used a 3.05% rate, then submitted $1,350 although its own arithmetic produced about $1,248. Indiana subtracts only the $1,000 base exemption and the resulting liability is $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model started from an unexplained federal AGI of $40,829 instead of $42,909.38 and also used a 3.05% rate. Subtracting the $1,000 exemption from the correct AGI gives taxable income of $41,909.38." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented a $3,000 deduction for child-support payments and a $1,500 disability exemption, neither of which belongs in this calculation. It also used a 3.0% rate instead of applying the 2026 rate to $41,909.38." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used a 2.79% flat rate, which is not the rate used for this 2026 Indiana liability, and supplied no valid taxable-income derivation. The traced base is $41,909.38 and its tax is $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI by $2,080, using $40,829 instead of $42,909.38. The listed health premiums do not create that reduction in the traced Indiana calculation; after the $1,000 exemption, taxable income is $41,909.38." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly derived taxable income of about $41,909 but applied a 3.05% rate. Applying the applicable 2026 Indiana rate to the precise $41,909.38 base yields $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI by $2,080 and then used a 3.0% rate. Indiana taxable income is $41,909.38 after only the $1,000 base exemption, not $39,829." -us,scenario_015,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model deducted the full $4,510 capital loss instead of the $3,000 loss allowed in federal AGI and omitted the $49 IRA deduction, producing the wrong AGI. It also used 3.0%; the correct taxable base is $41,909.38 and the liability is $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer implies unsupported state deductions reduced the tax base well below the traced $41,909.38. Indiana uses $42,909.38 of AGI less the single $1,000 exemption, yielding $1,236.33 rather than $1,018." -us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated deductions, contributions, and unspecified credits as eliminating the entire Indiana liability. The household retains $41,909.38 of Indiana taxable income after the $1,000 exemption, producing $1,236.33 before refundable credits." -us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly derived taxable income of about $41,909 but applied an incorrect 3.025% rate. The applicable 2026 computation on the precise $41,909.38 base yields $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model subtracted two separate $1,000 amounts from AGI, duplicating the Indiana base exemption. Only one $1,000 exemption applies, so taxable income is $41,909.38 rather than $40,909." -us,scenario_015,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated Indiana as applying a standard deduction to wages and never derived the federal-AGI starting point or the actual state exemption. Indiana instead starts from $42,909.38 and subtracts the $1,000 base exemption, producing $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model correctly identified the approximately $41,909 taxable-income base but used an incorrect 2.85% rate. The applicable 2026 rate applied to $41,909.38 produces $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract." -us,scenario_015,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used a 3.05% rate and did not specify a reproducible taxable-income base, despite submitting a number lower than 3.05% of the traced base. The correct calculation taxes $41,909.38 and yields $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the $960 workers’ compensation payment as taxable, deducted the full $4,510 capital loss instead of applying the $3,000 federal limit, omitted the $1,000 Indiana exemption, and used a 3.05% rate. Federal AGI is $42,909.38, Indiana taxable income is $41,909.38, and the resulting tax is $1,236.33." +us,scenario_014,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the standard deduction and a personal exemption reduce taxable wages to zero. No personal exemption applies, and subtracting $32,200 from $88,927.65 leaves $56,727.65 taxable." +us,scenario_014,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model improperly included the spouse's veterans benefits in AGI and invented two children and a $4,000 Child Tax Credit despite the prompt listing no children. It then submitted a negative value that does not match its own stated $4,616.80 tax-after-credit calculation; nonrefundable credits cannot reduce this output below zero." +us,scenario_014,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model invented CTC or ODC eligibility without any listed dependents and incorrectly claimed that deductions reduced taxable income to zero. The $32,200 standard deduction leaves $56,727.65 taxable, and no listed fact supports a nonrefundable credit." +us,scenario_014,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable output for federal_refundable_credits and therefore failed the required submission contract. +us,scenario_014,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model awarded refundable CTC despite the household having no qualifying children and awarded EITC despite approximately $183,584 of joint income, which exceeds the childless EITC phase-out limit. Those two credits both equal zero, so their asserted $3,067 total has no eligible credit pathway." +us,scenario_014,head_medicaid_eligible,qwen3.8-max,llm_error,health_coverage,False,"The model treated the head's lack of earned income as though Medicaid tested only the head's individual earnings and incorrectly declared household MAGI below the adult limit. The applicable MAGI is approximately $183,584, or 4.11 times FPL, and the head qualifies through no disability, SSI, dependent, or other categorical pathway." +us,scenario_014,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model correctly applied Social Security and Medicare taxes to the spouse’s wages but invented a $6.21 West Virginia employee state payroll tax at an estimated 0.2% rate. No such employee tax applies, so the payroll-tax total is the two traced federal components, $5,513.51 plus $1,289.45." +us,scenario_014,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly deducted the $14,717 employer-sponsored insurance premium from gross wages to create a $74,211 FICA base. The premium input does not establish a Section 125 exclusion from payroll-taxable wages, so Social Security and Medicare apply to the full $88,928 wage amount." +us,scenario_014,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly treated the reported $14,717 employer-sponsored insurance premium as a pre-tax FICA exclusion and reduced taxable wages to $74,211. The payroll-tax computation uses the full $88,928 of wages, yielding $5,513.51 of Social Security tax and $1,289.45 of Medicare tax." +us,scenario_014,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted the $14,717 insurance premium from wages without any fact establishing that it reduces Social Security or Medicare wages. Applying the employee FICA components to the full $88,928 wage base produces $6,802.96." +us,scenario_014,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages from $88,928 to $74,211 by deducting the employer-sponsored insurance premium. PolicyEngine retains the full wages in the payroll-tax base and computes $5,513.51 of Social Security tax plus $1,289.45 of Medicare tax." +us,scenario_014,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model incorrectly assumed the $14,717 employer-sponsored insurance premium was paid pre-tax for FICA purposes. The stated household facts do not remove that amount from payroll-taxable wages, so the full $88,928 wage amount is subject to employee Social Security and Medicare taxes." +us,scenario_014,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model’s explanation identifies the correct 7.65% employee FICA computation on $88,928 with no Additional Medicare Tax, but its submitted $13,591.63 does not follow that computation and effectively double-counts the liability. The stated calculation yields the traced total of $6,802.96." +us,scenario_014,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model submitted an unsupported estimate that does not equal the stated 6.2% Social Security and 1.45% Medicare taxes on the spouse’s $88,928 of wages. Those components are $5,513.51 and $1,289.45, totaling $6,802.96." +us,scenario_014,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly deducted the $14,717 employer-sponsored insurance premium from the spouse’s payroll-taxable wages. The full $88,928 wage amount remains subject to employee Social Security and Medicare taxes, producing $6,802.96." +us,scenario_014,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, violating the required structured-output contract." +us,scenario_014,payroll_tax,minimax-m3,llm_error,other,False,"The model’s own component calculation totals approximately $6,803, but it submitted $6,581 instead. The traced components are $5,513.51 of Social Security tax and $1,289.45 of Medicare tax, which sum to $6,802.96." +us,scenario_014,payroll_tax,qwen3.8-max,llm_error,other,False,"The model’s explanation computes approximately $6,803 from Social Security and Medicare taxes, but its submitted $8,075.02 includes an unexplained extra amount. With no Additional Medicare Tax or mandatory West Virginia employee payroll tax, the two federal components total $6,802.96." +us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reached approximately $84,928 of taxable income but applied an incorrect projected rate schedule. Its own bracket arithmetic gives about $3,255, then it abandoned that result and raised the answer to $3,374 through unsupported rounding instead of applying the schedule that yields $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the $14,717 employer-sponsored insurance premium from the reported gross wages and also treated child support as deductible. West Virginia AGI is $88,927.65 and taxable income after the $4,000 personal exemption is $84,927.65, not approximately $74,411." +us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly identified approximately $84,928 of taxable income, but used the wrong rate schedule and then subtracted an unidentified nonrefundable credit despite stating that the family tax credit was unavailable. No such reduction takes the computed tax to $2,877." +us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used approximate and internally changing marginal rates, then imposed an unsupported further rate-cut adjustment. Applying the applicable 2026 schedule to $84,927.65 produces $3,092.19, not $2,849." +us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model reached the correct taxable-income base but asserted that unspecified nonrefundable credits reduced the tax to $1,655. No applicable credit supports that reduction, and the 2026 bracket calculation yields $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model repeatedly calculated tax near $3,360 using an obsolete rate schedule, then cut it to $1,919 by inventing additional rate triggers and a low-income or other credit that it had already recognized did not apply. The applicable 2026 schedule on $84,927.65, with no such credit, yields $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly derived approximately $84,928 of taxable income but used an imprecise range of post-cut rates rather than the applicable 2026 bracket parameters. That rate approximation produced $3,200 instead of $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced AGI to $74,211 and then subtracted a nonexistent $15,400 West Virginia standard deduction. The correct base is $88,927.65 less only $4,000 of personal exemptions, leaving $84,927.65." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model treated the $14,717 employer-sponsored premium as a deduction from the reported wages, reducing AGI to $74,211. The traced AGI is $88,927.65, so taxable income is $84,927.65 rather than $70,211." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer supplies no computation and its $4,125 estimate applies an excessive effective rate to the household's taxable wages. West Virginia taxes $84,927.65 under its graduated 2026 schedule, producing $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $74,211 as AGI, reflecting subtraction of the employer-sponsored premium from the reported wages. AGI is $88,927.65 and taxable income after two personal exemptions is $84,927.65." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the wage-derived AGI to $74,211 before applying the personal exemptions. The correct taxable-income base is $84,927.65, after subtracting $4,000 from AGI of $88,927.65." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly started from AGI of $74,211, thereby deducting the listed employer-sponsored premium from wages. West Virginia instead starts from $88,927.65 and taxes $84,927.65 after the $4,000 exemption." +us,scenario_014,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented an $8,000 West Virginia standard deduction and then applied the top marginal rate to the entire remaining base. West Virginia subtracts only the $4,000 personal exemption here and applies graduated brackets to $84,927.65." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The answer gives no supporting computation, and $2,331 is inconsistent with applying the 2026 West Virginia joint-filer brackets to taxable income of $84,927.65. That bracket calculation produces $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model invented deductions and nonrefundable items that fully eliminated the tax. The household has $84,927.65 of West Virginia taxable income and no applicable nonrefundable credit that reduces the resulting $3,092.19 liability to zero." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly derived approximately $84,928 of taxable income but used the wrong 2.22%, 2.96%, 3.33%, 4.44%, and 4.82% schedule. The applicable 2026 parameters produce $3,092.19 rather than $3,255.03." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The taxable-income derivation was essentially correct, but the model applied an incorrect 2026 West Virginia rate schedule. Taxing $84,927.65 under the applicable schedule yields $3,092.19, not $3,217.53." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly identified approximately $84,928 of taxable income but applied a projected rate schedule that overstates the liability. The applicable 2026 brackets yield $3,092.19 rather than $3,255.03." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the wage base and two personal exemptions but used the wrong West Virginia rate schedule. Applying the applicable 2026 brackets to $84,927.65 yields $3,092.19 rather than $3,255." +us,scenario_014,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model invoked a standard deduction that does not reduce West Virginia taxable income to zero. After the $4,000 personal exemption, $84,927.65 remains taxable and produces $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly reached approximately $84,928 of taxable income but used a 2024-style schedule as the 2026 schedule. Those obsolete rates overstate the liability at $3,460 instead of $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model first understated AGI as $74,211 and then incorrectly subtracted the federal standard deduction in computing West Virginia taxable income. West Virginia begins with AGI of $88,927.65 and subtracts the $4,000 state personal exemption, leaving $84,927.65." +us,scenario_014,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used approximately the correct taxable-income base but substituted approximate brackets that overstate the tax. The applicable 2026 schedule on $84,927.65 produces $3,092.19, not $3,422." +us,scenario_014,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the $14,717 employer-sponsored premium from wages, invented an $8,000 West Virginia standard deduction, and treated the tax as a flat 4.5%. The correct computation taxes $84,927.65 under graduated brackets." +us,scenario_014,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly reached taxable income of approximately $84,928 but used an obsolete bracket base and 5.12% top marginal rate. The applicable 2026 schedule yields $3,092.19 rather than $3,459.81." +us,scenario_014,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the personal exemptions as large enough to erase all wage income. The exemptions total only $4,000, leaving $84,927.65 of taxable income and $3,092.19 of tax." +us,scenario_014,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model wrongly deducted the employer-sponsored premium from wages, applied a reversed and incorrect rate schedule, and invented a 4% family tax credit at this income. The correct base is $84,927.65, and no such credit reduces the bracket tax of $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly applied the federal standard deduction to the West Virginia base and invented nonrefundable credits that eliminated the remaining liability. West Virginia taxes $84,927.65 after its $4,000 personal exemption, yielding $3,092.19 before refundable credits." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own AGI, standard deduction, taxable income, and bracket calculation produce about $2,969, but it submitted $3,103 instead. The submitted value does not follow from its stated computation." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model included nontaxable workers' compensation, used a stale $14,600 standard deduction instead of $16,100, and failed to enforce the $3,000 annual capital-loss limit consistently. Its final $3,509 also contradicts its own intermediate tax of $3,094.28." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly derived taxable income near $26,809 and tax near $2,974, but submitted $3,268. Nothing in its stated calculation adds the unexplained $294 difference." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model treated the listed $45,000 gross wages as already reduced by the traditional 401(k) contribution and therefore omitted the $1,041.93 pre-tax wage reduction. That inflated taxable income from $26,809.38 to $27,851 and tax from $2,969.13 to $3,094." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model incorrectly treated the $4,510 capital loss as fully deductible despite the $3,000 limit and omitted the traditional 401(k) reduction, then calculated about $2,841. It nevertheless submitted $4,172, a value unsupported by any step in its reasoning." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model ultimately substituted the 2025 $15,000 standard deduction for the 2026 $16,100 amount. This raised taxable income by $1,100 and led to $3,111 instead of applying the 2026 schedule to $26,809.38." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model first computed roughly $3,011 and then an estimated $2,876 after a Saver's Credit, but submitted $1,750 based on unspecified additional adjustments. Disability creates neither an extra standard deduction at age 36 nor a child credit, so the unexplained reduction has no valid computation step." +us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted the $2,080 health-insurance premium from gross wages even though this input is not an additional federal above-the-line deduction, and it used a $15,000 rather than $16,100 standard deduction. Those errors produced the wrong AGI and taxable income." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset, replaced the $16,100 standard deduction with an $8,300 deduction plus personal exemption, and applied a 15% second bracket. The 2026 computation instead uses the $16,100 standard deduction and a 12% rate above $12,400." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied no derivation beyond an approximate AGI and invoked unspecified nonrefundable credits. With taxable income of $26,809.38 and no applicable nonrefundable credits, the bracket calculation yields $2,969.13 rather than $1,541." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the separately listed $2,080 employer-sponsored health premium from wages and assumed a TCJA sunset with a personal exemption. The applicable computation uses taxable income of $26,809.38 under the continuing 2026 standard-deduction regime." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used the 2025 $15,000 standard deduction instead of the 2026 $16,100 amount and used an $11,600 bracket threshold instead of $12,400. These stale parameters raised its result to $3,117." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by the $2,080 health-insurance premium, producing AGI of $40,829 instead of the traced income calculation. It also used a $15,700 standard deduction rather than $16,100." +us,scenario_015,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used estimated parameters—a $15,400 standard deduction and a $12,247 first-bracket ceiling—instead of the 2026 values of $16,100 and $12,400. The correct parameters applied to $26,809.38 yield $2,969.13." +us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model assumed unspecified nonrefundable credits eliminated the tax, but this single filer has no qualifying dependents or other facts generating such credits. The standard deduction leaves $26,809.38 taxable, so regular tax remains $2,969.13." +us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated child-support payments and health-related amounts as deductions or allowances even though child support is nondeductible and the health inputs do not create an above-the-line deduction here. It also gave no bracket computation capable of producing $5,199 from taxable income of $26,809.38." +us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model stated the correct taxable income, $16,100 deduction, $12,400 threshold, and 10%/12% rates, but its submitted arithmetic is wrong. Those stated inputs calculate to $1,240 plus $1,729.13, or $2,969.13, not $3,211.08." +us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model netted all $1,351 of desired retirement contributions from wages and used the full $4,510 capital loss. Only the deductible traditional amounts apply, and the ordinary-income capital-loss deduction is capped at $3,000; applying the traced deductions produces taxable income of $26,809.38." +us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed the pre-TCJA standard deduction, personal exemption, and 15% bracket returned in 2026. The applicable 2026 rules instead provide a $16,100 standard deduction and tax the income above $12,400 at 12%." +us,scenario_015,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly reverted to pre-2018 deductions, a personal exemption, and a 15% second bracket. The 2026 single schedule uses a $16,100 standard deduction and 10%/12% brackets, producing $2,969.13." +us,scenario_015,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the outstanding mortgage balance and estimated SALT rather than using listed deductible expenses. A mortgage balance alone does not establish annual interest paid, so itemized deductions do not displace the $16,100 standard deduction." +us,scenario_015,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. +us,scenario_015,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model assumed disability and unspecified credits reduced the liability to zero. Disability alone supplies no credit that offsets the regular tax here, and the $16,100 standard deduction still leaves $26,809.38 taxable." +us,scenario_015,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model incorrectly included $960 of nontaxable workers' compensation and used a $15,750 standard deduction, but its own resulting tax was $3,137.16. It then submitted $2,104 without identifying any credit or computation that subtracts $1,033.16." +us,scenario_015,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model treated the one-person household as married filing jointly, failed to subtract the traditional 401(k) and IRA deductions, and invented dependent and child-related credits despite there being no dependent. It also combined incompatible tax amounts to reach $4,072.21 rather than following a single liability calculation." +us,scenario_015,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no parseable value for federal_refundable_credits, violating the required output contract." +us,scenario_015,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a $1,000 refundable Child Tax Credit despite the household having no qualifying child, and it awarded $1,735.80 of EITC even though this childless adult's approximately $43,450 income exceeds the applicable childless EITC range. Both components are $0, so their sum is $0." +us,scenario_015,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented qualification through Indiana’s Aged, Blind, and Disabled or working-disabled buy-in pathways from the disability and work facts without applying their actual categorical and financial requirements. The engine assigns category NONE: the head receives no SSI, qualifies through no disability-related pathway, and has MAGI at 2.69 times FPL, so the asserted buy-in eligibility does not exist." +us,scenario_015,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated Indiana Medicaid expansion and the disability flag as sufficient for eligibility without identifying or testing a qualifying category. At age 36 with MAGI at 2.69 times FPL and no qualifying disability-related pathway, the head is assigned Medicaid category NONE." +us,scenario_015,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model incorrectly treated the generic disability flag as independently sufficient for Medicare eligibility. At age 36, the head needs a qualifying under-65 Medicare pathway, such as SSDI entitlement followed by the required waiting period or end-stage renal disease, none of which is present." +us,scenario_015,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model collapsed Medicare's under-65 eligibility requirements into a simple disabled-adult rule. A reported disability without SSDI-based Medicare entitlement, completion of the waiting period, or another qualifying condition does not make this 36-year-old eligible." +us,scenario_015,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly treated the $2,080 of health insurance premiums as a pre-tax cafeteria-plan deduction from FICA wages. The facts only report premium spending, so Social Security and Medicare taxes apply to the full $45,000 wage amount, not $42,920." +us,scenario_015,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly subtracted the $2,080 health premium from FICA wages without any fact establishing that it was paid through a pre-tax employer plan. The correct payroll-tax base is the full $45,000, producing $2,790 of Social Security tax and $652.50 of Medicare tax." +us,scenario_015,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model treated the listed health insurance premiums as pre-tax deductions from Social Security and Medicare wages. No pre-tax payroll treatment is specified, so both FICA components use the full $45,000 wage base." +us,scenario_015,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model derived the correct components—$2,790 of Social Security tax and $652.50 of Medicare tax—but added them incorrectly. Their sum is $3,442.50, not $3,445.50." +us,scenario_015,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's stated Social Security amount of $2,790 and rounded Medicare amount of $653 already sum to $3,443, not $3,419. Using the exact Medicare calculation of $652.50 yields total payroll tax of $3,442.50." +us,scenario_015,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll-tax output or explanation. It therefore failed the required structured-output contract rather than completing the FICA calculation. +us,scenario_015,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model correctly calculated $3,442.50 of employee FICA and correctly stated that Indiana adds no mandatory employee state payroll tax, then replaced that total with $3,495 based on nonexistent county-tax adjustments. County income taxes do not belong in this payroll-tax output." +us,scenario_015,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model improperly deducted the $1,042 traditional 401(k) contribution from the FICA wage base; traditional 401(k) deferrals remain subject to Social Security and Medicare taxes. Its submitted $3,425.67 also contradicts its own stated component total of $3,362.79, while applying both rates to the full $45,000 yields $3,442.50." +us,scenario_015,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived the correct $41,909 taxable-income base and 2.95% rate, which produce $1,236.33, but then discarded that arithmetic and submitted $1,234 as an unsupported rounding adjustment." +us,scenario_015,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an obsolete 3.15% rate instead of Indiana's 2026 rate of 2.95%. It also deducted the full $4,510 capital loss, included workers’ compensation in its gross-income arithmetic, and invoked unspecified disability relief rather than applying the traced $42,909.38 AGI less the $1,000 exemption." +us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented a $500 disability exemption, reducing the traced taxable income below $41,909.38, and used 3.05% instead of 2.95%. Its submitted $1,352 also contradicts every calculation stated in its explanation." +us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $1,042 traditional 401(k) contribution when constructing AGI, producing $43,951 instead of $42,909.38. It then submitted $1,392 even though neither its stated 2.95% nor 3.05% rate produces that amount." +us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an incorrect 2.85% Indiana rate instead of 2.95% and began from an unsupported AGI of about $42,441. The traced computation taxes $41,909.38 after the sole $1,000 exemption." +us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model correctly derived federal AGI but invented both a $1,000 disability exemption and a $2,080 Indiana health-insurance-premium deduction. Only the $1,000 base exemption is subtracted, so the 2.95% rate applies to $41,909.38 rather than $38,829." +us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented an additional $1,000 disability exemption and used 3.05% instead of 2.95%. It then submitted $1,350 despite its own stated calculation yielding about $1,248." +us,scenario_015,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model started from an unexplained $40,829 AGI instead of $42,909.38 and used 3.05% instead of 2.95%. Indiana taxes $41,909.38 after the $1,000 exemption." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented a $3,000 child-support deduction and a $1,500 disability exemption, neither of which appears in this Indiana tax base. It also used 3.0% rather than the 2026 rate of 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used an incorrect 2.79% Indiana rate. The correct computation applies 2.95% to $41,909.38 after the $1,000 base exemption." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used $40,829 as federal AGI instead of $42,909.38. Although it selected the correct 2.95% rate and $1,000 exemption, applying them to the understated base produced $1,175 instead of $1,236.33." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly identified the $41,909 taxable-income base but applied 3.05% instead of Indiana's 2026 rate of 2.95%. Applying 2.95% yields $1,236.33." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI as $40,829 rather than $42,909.38 and used 3.0% instead of 2.95%. The correct base after the exemption is $41,909.38." +us,scenario_015,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model deducted the entire $4,510 capital loss rather than the $3,000 federal limit and omitted the traditional 401(k) and IRA adjustments from its stated AGI calculation. It also used 3.0% instead of the 2026 Indiana rate of 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer implies an unsupported taxable-income reduction beyond Indiana's $1,000 base exemption. Federal AGI of $42,909.38 less that exemption leaves $41,909.38, whose 2.95% tax is $1,236.33." +us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed deductions and credits eliminated the liability. Indiana retains $41,909.38 of taxable income after the $1,000 exemption, producing $1,236.33 before refundable credits." +us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly derived the taxable-income base but used a fabricated 3.025% rate instead of Indiana's 2026 rate of 2.95%. The correct rate applied to $41,909.38 yields $1,236.33." +us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model subtracted two separate $1,000 amounts from federal AGI, duplicating Indiana's base exemption. Only one $1,000 exemption applies, leaving $41,909.38 to be taxed at 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated Indiana as allowing an unspecified standard deduction and estimated from gross wages rather than constructing Indiana AGI. The correct base is federal AGI of $42,909.38 less the $1,000 exemption, taxed at 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model correctly identified taxable income of about $41,909 but used a 2.85% rate instead of Indiana's 2026 rate of 2.95%. The correct multiplication produces $1,236.33." +us,scenario_015,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not be parsed or evaluated as a substantive estimate." +us,scenario_015,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used 3.05% instead of Indiana's 2026 rate of 2.95% and did not state a reproducible taxable-income base. The traced base is $41,909.38, producing $1,236.33." +us,scenario_015,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated nontaxable workers’ compensation as part of federal AGI, used the full $4,510 capital loss instead of the $3,000 limit, omitted Indiana's $1,000 base exemption, and applied 3.05% instead of 2.95%. The correct sequence yields $42,909.38 of AGI and $41,909.38 of taxable income." +us,scenario_015,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model substituted federal taxable income after a federal-style deduction for Indiana's federal-AGI starting point. Indiana instead starts from $42,909.38, subtracts only its $1,000 exemption, and applies 2.95% to $41,909.38." us,scenario_015,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_016,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model correctly derived that medical itemization reduces taxable income and tax to zero, then discarded that calculation and submitted an unexplained standard-deduction result. The taxpayer itemizes the much larger deductible medical expenses instead of using the standard deduction." -us,scenario_016,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,The model ignored the medical itemized deduction that reduces taxable income to zero. It also invented mortgage interest from a mortgage balance and a Saver's Credit without any listed mortgage-interest payment or retirement contribution. -us,scenario_016,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model used the standard deduction and omitted the substantially larger medical itemized deduction. Its submitted $1,503 also does not follow its own stated $1,723.60 tax calculation." -us,scenario_016,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,The model applied the 2024 standard deduction and brackets as a proxy and never compared the standard deduction with deductible medical expenses after the 7.5% AGI floor. Medical itemization reduces taxable income to zero. -us,scenario_016,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,The model mechanically subtracted the student-loan-interest adjustment and standard deduction while omitting the larger medical itemized deduction. Applying that deduction after its AGI floor leaves no taxable income. -us,scenario_016,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model allowed deductions to drive taxable income below zero and then treated the excess deductions or nonrefundable credits as a negative tax payment. Taxable income floors at zero, and nonrefundable credits reduce liability only to zero." -us,scenario_016,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model chose the standard deduction without applying the larger medical itemized deduction, which reduces taxable income to zero. It also deducted the full $3,000 of student-loan interest instead of the applicable capped amount reflected in AGI of $32,137.94." -us,scenario_016,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model invented a $3,995 childless-worker EITC maximum and treated $34,638 of wages as within the maximum-credit range. Under the 2026 childless EITC rules, a single filer with no qualifying children is phased out before this income level, so EITC is $0 and no other refundable federal credit applies." -us,scenario_016,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model ignored the listed $34,638 of gross wages and treated payroll tax as zero unless a separate payroll-tax input or Florida state payroll tax was listed. Employee-side payroll_tax includes federal FICA on wages, so the head owes Social Security tax and Medicare tax even with no state payroll tax." -us,scenario_016,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model identified the correct FICA rates and even computed a total near $2,648.40, then submitted $2,157.70, which is inconsistent with its own stated Social Security-plus-Medicare calculation. It failed the final arithmetic/answer-transfer step for employee-side FICA on $34,638 of wages." -us,scenario_016,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly applied the wage base, Social Security rate, Medicare rate, Additional Medicare threshold, and Florida state payroll-tax rule, then replaced its own approximately $2,650 total with an unsupported rounded value of $2,674. The error is an unjustified final adjustment after the correct component calculation." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted $1,625 of auto-loan interest from taxable income. The trace applies only the $16,100 standard deduction, leaving taxable income of $45,492.09 rather than $43,867." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction and incorrect bracket thresholds instead of the 2026 $16,100 single standard deduction and applicable 10% and 12% brackets. Its submitted $4,903 also contradicts its own stated $5,716 calculation." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model explicitly derived the correct bracket tax of about $5,211, then replaced it with $6,754 without any computational step or adjustment supporting the change. No modification applies after the stated regular-tax calculation." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly identified taxable income of about $45,492 and components of $1,240 plus $3,971, but added them incorrectly. Those components total about $5,211, not $6,033." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model used outdated projected deduction and bracket values and then incorrectly allowed a $200 Saver's Credit. At AGI $61,592.09 no Saver's Credit applies in the trace, while the $16,100 standard deduction and 2026 brackets yield $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model first calculated regular tax near $5,216, then reduced it to $4,913 for an unspecified retirement-savings credit. No Saver's Credit applies at this AGI, and the exact 2026 brackets produce $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model treated the 2026 system as a post-TCJA sunset regime with a $7,700 standard deduction, a personal exemption, and 15% brackets. It also deducted the listed $9,000 health premium from AGI even though the trace includes no such pre-tax deduction; current 2026 rules instead give AGI of $61,592.09 and a $16,100 standard deduction." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly reverted to pre-TCJA-style deductions, a personal exemption, and 15% and 25% rates. The applicable 2026 computation uses the $16,100 single standard deduction and 10% and 12% brackets on $45,492.09." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used a $14,600 standard deduction and omitted both the $50 taxable interest and $288.51 traditional IRA deduction from its stated taxable-income derivation. The correct taxable income is $45,492.09, whose bracket tax is $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed a pre-TCJA regime with a combined estimated standard deduction and personal exemption of $13,790. The 2026 calculation instead uses a $16,100 standard deduction with no personal exemption, followed by the applicable 10% and 12% brackets." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly deducted $9,000 of health premiums from AGI and applied a sunset regime with a personal exemption and 15% marginal rate. The trace gives AGI of $61,592.09 and taxes $45,492.09 under the 2026 10% and 12% brackets." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $15,700 standard deduction and a $12,275 first-bracket ceiling. The applicable standard deduction is $16,100, and the exact 2026 bracket calculation on $45,492.09 yields $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $15,400 standard deduction and estimated bracket threshold rather than the enacted 2026 values. The $16,100 deduction lowers taxable income to $45,492.09, producing $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented a qualifying child and subtracted a nonrefundable Child Tax Credit. The household contains only the 53-year-old head, so no CTC applies and regular tax remains $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model assumed unspecified nonrefundable credits eliminated the tax despite no qualifying credit facts. The stated income and deductions leave $45,492.09 taxable income and $5,211.05 of tax." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly claimed a $6,760 qualified-overtime deduction. The trace contains no overtime deduction; after only the 401(k), IRA, and standard-deduction adjustments, taxable income is $45,492.09." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model reached the correct AGI, standard deduction, and taxable-income framework but applied the 2026 bracket arithmetic incorrectly. The exact tax on $45,492.09 is $5,211.05, not $5,220.54." -us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model rounded the bracket calculation to an unsupported $5,200 estimate. Applying the exact 2026 thresholds and rates to $45,492.09 yields $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted the two $9,000 premium fields as $18,000 of medical expense, invented a SALT deduction, and applied a personal exemption after itemizing. The correct computation takes the $16,100 standard deduction with no personal exemption, leaving $45,492.09 taxable income." -us,scenario_018,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $15,375 estimated standard deduction rather than the 2026 $16,100 amount. The larger applicable deduction leaves $45,492.09 taxable income and $5,211.05 of tax." -us,scenario_018,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. -us,scenario_018,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The zero answer omits the regular income-tax computation entirely. AGI of $61,592.09 less the $16,100 standard deduction leaves $45,492.09 taxable income, and no nonrefundable credits reduce the resulting $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $288.51 traditional IRA deduction from AGI and used an estimated $15,750 standard deduction. The correct AGI is $61,592.09 and the applicable $16,100 deduction produces taxable income of $45,492.09." +us,scenario_016,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model correctly calculated that itemizing the medical expenses reduces taxable income and tax to zero, but then discarded that result and submitted a standard-deduction computation. Its $1,523 answer contradicts its own conclusion that the taxpayer optimally itemizes." +us,scenario_016,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model ignored the itemized medical-expense deduction that eliminates taxable income. It also invented mortgage interest from the mortgage balance and a Saver's Credit without any listed interest payment or retirement contribution, neither of which can replace the required computation from the supplied inputs." +us,scenario_016,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model applied the standard deduction and omitted the much larger itemized medical-expense deduction, leaving taxable income when it should be zero. Its submitted $1,503 also does not follow from its own stated tax calculation of $1,723.60." +us,scenario_016,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,The model used 2024 parameters as a proxy for 2026 and applied the standard deduction while omitting the medical-expense itemized deduction. The medical deduction exceeds the remaining income and reduces taxable income and regular tax to zero. +us,scenario_016,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model selected the standard deduction and ignored the substantially larger deduction for medical expenses above 7.5% of AGI. Itemizing those listed expenses reduces taxable income to zero, so no income reaches the 10% or 12% brackets." +us,scenario_016,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly converted deductions exceeding income into a negative tax and described unused nonrefundable credits as an excess credit position. Taxable income and federal income tax before refundable credits are floored at zero, and the listed medical amounts cannot produce a negative $156,300 liability." +us,scenario_016,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model applied only a standard deduction and omitted the larger itemized medical-expense deduction, which reduces taxable income to zero. It also deducted the full $3,000 of student-loan interest instead of applying the statutory $2,500 cap reflected in the traced AGI." +us,scenario_016,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model invented a $3,995 childless EITC maximum and treated $34,638 as lying in a maximum-credit plateau. The childless EITC is fully phased out at this income, so the EITC and total federal refundable credits are $0." +us,scenario_016,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model applied a partial childless EITC phaseout and stopped at $360 instead of recognizing that $34,638 exceeds the phaseout endpoint. With no other refundable-credit pathway, total federal refundable credits are $0." +us,scenario_016,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly applied a Medicaid expansion threshold to Florida, which has no expansion pathway for a nondisabled, nonpregnant adult without a qualifying dependent category. It also understated MAGI by treating the student-loan-interest deduction as sufficient to reach that nonexistent pathway; the engine places MAGI at 2.01 times FPL and assigns Medicaid category NONE." +us,scenario_016,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model treated payroll tax as requiring a separately listed payroll-tax input and ignored that employee FICA is automatically imposed on the listed $34,638 of wages. It omitted both the 6.2% employee Social Security tax and the 1.45% employee Medicare tax." +us,scenario_016,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model stated both applicable FICA components and even obtained an intermediate total near $2,648.40, but then submitted $2,157.70 without adding Social Security and Medicare correctly. The traced components are $2,147.55 and $502.25, which sum to $2,649.80." +us,scenario_016,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly identified every applicable payroll-tax component and computed a total near $2,650, then replaced that result with an unsupported $2,674 'minor adjustment.' No adjustment or rounding rule adds $24.20; the component amounts sum directly to $2,649.80." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $1,625 of auto-loan interest from taxable income. That deduction does not apply to the supplied vehicle-loan facts, so taxable income remains $45,492.09 rather than $43,867." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction and incorrect bracket thresholds instead of the 2026 $16,100 deduction and applicable brackets. It then submitted $4,903 even though its own stated calculation produced $5,716, so its final value also contradicts its reasoning." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly derived taxable income of about $45,492 and tax of about $5,211, then replaced that result with $6,754 without any computation supporting the adjustment. Its submitted value contradicts its own completed bracket calculation." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly computed $1,240 plus approximately $3,971, which totals about $5,211, but reported $6,033. This is an arithmetic and answer-transcription error after correctly identifying taxable income and the bracket structure." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model used an obsolete $15,000 standard deduction and incorrectly awarded a $200 Saver's Credit at AGI of $61,592. The correct $16,100 standard deduction and 2026 credit rules leave tax of $5,211.05 with no nonrefundable credit adjustment." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model's stated brackets produced about $5,216, but it then subtracted an unspecified Saver's Credit to reach $4,913. No nonrefundable credit applies, and the precise 2026 bracket calculation yields $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model treated the $9,000 health-insurance premium as pre-tax despite the prompt not identifying it as a payroll exclusion, and it assumed a TCJA sunset with a $7,700 standard deduction, personal exemption, and 15% bracket. The governing 2026 rules instead produce AGI of $61,592.09, a $16,100 standard deduction, and $5,211.05 of tax." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed that the TCJA rules sunset for 2026 and substituted a smaller standard deduction, restored personal exemption, and pre-TCJA 15% and 25% brackets. The applicable $16,100 standard deduction and 10%/12% brackets yield $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used a $14,600 standard deduction and omitted both the $50 taxable interest and $289 deductible traditional IRA contribution from its taxable-income derivation. The complete computation gives taxable income of $45,492.09 and tax of $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied projected pre-TCJA deductions, a personal exemption, and reverted rate brackets. The 2026 calculation uses a $16,100 single standard deduction with the applicable 10% and 12% brackets, not the assumed pre-TCJA regime." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model wrongly excluded the $9,000 insurance premium from income and assumed restored pre-TCJA deductions and a 15% bracket. The premium is not specified as pre-tax, and the applicable 2026 standard deduction and brackets produce $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $15,700 standard deduction and incorrect 2026 bracket cutoff instead of the $16,100 deduction and engine parameters. This overstated taxable income and tax, which are $45,492.09 and $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model estimated the single standard deduction at $15,400 and used an incorrect $12,260 first-bracket ceiling. Applying the $16,100 deduction and exact 2026 brackets yields $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model invented a qualifying child and subtracted a nonrefundable Child Tax Credit even though the household contains only the 53-year-old head. With no dependent and no applicable nonrefundable credit, the bracket tax remains $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model zeroed the liability by assuming unspecified nonrefundable credits exhausted the tax. No such credit applies, and $45,492.09 of taxable income produces $5,211.05 before refundable credits." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model invented a $6,760 qualified-overtime deduction even though the facts provide annual wages, hours, and a straight-time rate but no separately established deductible overtime amount. Without that unsupported deduction, taxable income is $45,492.09 and tax is $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly derived AGI and the $16,100 standard deduction but applied the 2026 brackets inaccurately. The exact bracket computation on $45,492.09 is $5,211.05, not $5,220.54." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model rounded an approximate $5.2 thousand estimate instead of applying the exact 2026 brackets to $45,492.09 of taxable income. That calculation yields $5,211.05 rather than $5,200." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model invented deductible SALT, counted $18,500 of medical expenses despite only $11,000 being listed across the relevant premium and expense fields, and restored a personal exemption under an inapplicable sunset assumption. The standard deduction is $16,100, no personal exemption is subtracted, and taxable income is $45,492.09." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $15,375 estimated standard deduction instead of the 2026 $16,100 amount. This overstated taxable income and produced $5,302 rather than $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model overstated deductible medical expenses and invented a state-tax itemized deduction, causing it to itemize instead of taking the $16,100 standard deduction. The standard deduction produces taxable income of $45,492.09 and tax of $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for the requested variable, so the required numeric value was missing." +us,scenario_018,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer ignores the positive taxable income remaining after the retirement deductions and $16,100 standard deduction. The stated inputs produce $45,492.09 of taxable income and $5,211.05 of federal tax before refundable credits." +us,scenario_018,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $289 traditional IRA deduction and used an estimated $15,750 standard deduction rather than $16,100. The correct deductions produce taxable income of $45,492.09 and tax of $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model double-counted retirement contributions in reducing taxable income to about $32,942 and then applied an unsupported $303 Saver's Credit. The 401(k) and IRA amounts reduce income only through their specified pre-tax and above-the-line treatment, leaving $45,492.09 taxable and no nonrefundable credit adjustment." us,scenario_018,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_018,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model computed the correct Social Security tax of $4,216.31 and Medicare tax of $986.07, then added a fabricated $49.41 rounding adjustment. The two stated components already sum to $5,202.38, so the submitted $5,251.79 is an arithmetic add-on error unsupported by the payroll tax rules." -us,scenario_018,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model incorrectly subtracted $9,000 of health insurance premiums from FICA wages, reducing the payroll tax base from $68,005 to $59,005. PolicyEngine applies employee Social Security and Medicare tax to the full listed wage amount here, producing $5,202.38 rather than 7.65% of the reduced base." -us,scenario_018,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model treated the listed employer-sponsored health insurance premiums as pre-tax payroll deductions that reduce Social Security and Medicare wages. The payroll tax calculation uses the full $68,005 wage amount, so subtracting $9,000 from the FICA base understated both the 6.2% Social Security tax and the 1.45% Medicare tax." -us,scenario_018,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model excluded federal FICA from the payroll_tax output and treated the absence of Arizona employee payroll taxes as making the output zero. The requested output explicitly includes employee Social Security and Medicare tax, so the employee wage earner owes $4,216.31 of Social Security tax and $986.07 of Medicare tax." -us,scenario_018,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct component formulas, $68,005 times 6.2% and $68,005 times 1.45%, but then submitted $4,089 instead of their sum. The Social Security wage base cap does not bind at $68,005, and the correct component total is $5,202.38." -us,scenario_018,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a payroll_tax value or explanation, so it failed the required output contract. The correct computation applies employee Social Security and Medicare tax to $68,005 of wages, yielding $5,202.38." -us,scenario_018,payroll_tax,minimax-m3,llm_error,other,False,"The model submitted zero for payroll_tax, which omits the federal employee FICA taxes included in the requested output. With $68,005 of wages, the household owes employee Social Security tax of $4,216.31 and employee Medicare tax of $986.07." +us,scenario_018,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly computed $4,216.31 of Social Security tax and $986.07 of Medicare tax, which sum exactly to $5,202.38, then invented a $49.41 “rounding adjustment.” No such adjustment applies." +us,scenario_018,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the listed $9,000 health-insurance premiums from FICA wages as though they were documented pre-tax employer-plan salary reductions. Applying 7.65% to the full $68,005 wage base yields $5,202.38." +us,scenario_018,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model treated the $9,000 health-insurance-premium input as a pre-tax payroll deduction and reduced FICA wages to $59,005. The facts do not designate those premiums as pre-tax salary reductions, so Social Security and Medicare taxes apply to all $68,005 of wages." +us,scenario_018,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model confused Arizona's lack of an additional mandatory employee state payroll tax with the absence of all employee payroll tax. Federal employee FICA still produces $4,216.31 of Social Security tax and $986.07 of Medicare tax." +us,scenario_018,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct component calculations—about $4,216 of Social Security tax and $986 of Medicare tax—but replaced their $5,202.38 sum with an unsupported $4,089 estimate. Neither the Social Security wage cap nor rounding reduces the total because $68,005 is below the cap." +us,scenario_018,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required output contract." +us,scenario_018,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer omits federal employee FICA on $68,005 of wages. The correct computation includes $4,216.31 of Social Security tax and $986.07 of Medicare tax, totaling $5,202.38." us,scenario_018,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_018,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived $45,842 of taxable income and approximately $1,146 of tax, then submitted $1,503 without any supporting computation. Its unrelated discussion of a federal auto-loan-interest deduction neither produces nor supports the submitted state-tax value." -us,scenario_018,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used obsolete progressive Arizona brackets instead of the 2.5% flat rate and used a $14,600 deduction instead of $15,750. It then submitted $2,175 despite its own bracket calculation producing $1,304.40 and incorrectly described medical expenses as a state tax credit." -us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model estimated the Arizona single standard deduction as $15,000 rather than $15,750. That left taxable income $750 too high and produced $1,165 instead of applying 2.5% to $45,842.09." -us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's stated AGI, approximate deduction, and 2.5% rate produce about $1,175, not its submitted $1,471. It also used a $14,600 deduction rather than Arizona's $15,750 amount." -us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model submitted $1,481 even though its final stated calculation was $1,137. It also substituted an estimated $16,100 standard deduction for the applicable $15,750 deduction." -us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used an estimated $14,600 standard deduction rather than the applicable $15,750 Arizona deduction. The correct subtraction from $61,592.09 gives $45,842.09 of taxable income, not $46,992." -us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's own detailed computation yielded approximately $1,175, but it arbitrarily adjusted the result to $1,450 for unspecified premium considerations. Health-premium considerations do not justify that adjustment, and the applicable $15,750 standard deduction yields $1,146.05." -us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $52,592 and used a nonexistent $7,700 Arizona standard deduction. The trace instead gives $61,592.09 of AGI and a $15,750 standard deduction, producing $45,842.09 of taxable income." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used a projected $14,600 standard deduction rather than Arizona's $15,750 deduction. This overstated taxable income by $1,150 and tax by $28.75." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $1,400 does not follow from the traced $61,592.09 AGI, $15,750 standard deduction, and 2.5% rate. It implies the model failed to calculate the applicable deduction and resulting $45,842.09 taxable income." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an estimated $13,790 standard deduction instead of $15,750. That overstated Arizona taxable income by $1,960 and produced an excessive tax estimate." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated Arizona AGI by $9,000, using $52,592 rather than $61,592.09. Although it also used the wrong $14,600 deduction, the principal error was removing $9,000 of health premiums from AGI when those premiums belong in the itemized-deduction comparison." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used a $15,700 standard deduction instead of $15,750. This left taxable income $50 too high and tax $1.25 too high." -us,scenario_018,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model made a decimal-place transcription error, submitting $11,548 after computing $46,192 × 2.5% as $1,154.80. It also used a $15,400 deduction instead of $15,750." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The unexplained $1,941 result does not apply the traced $15,750 standard deduction and 2.5% flat rate to $61,592.09 of AGI. Those inputs yield $45,842.09 of taxable income and $1,146.05 of tax." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model zeroed the liability merely because there were no dependents or major credits. Arizona's $45,842.09 of taxable income is subject to the 2.5% rate, and no listed nonrefundable credit eliminates the resulting $1,146.05." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model deducted $18,500 of medical and insurance expenses directly, even though Arizona compares allowable itemized deductions of $10,786.71 with the $15,750 standard deduction and selects the latter. Its claimed $43,092 taxable income also does not equal $61,592 minus $18,500, exposing an additional arithmetic error." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used a $16,100 standard deduction rather than $15,750. This understated taxable income by $350 and tax by $8.75." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used a $16,100 standard deduction rather than $15,750. The correct taxable income is $45,842.09, not $45,492." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $45,492 taxable-income estimate embeds a $16,100 deduction instead of the applicable $15,750 deduction. Applying the same 2.5% rate to $45,842.09 yields $1,146.05." -us,scenario_018,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model treated Arizona tax as progressive and applied an unspecified average rate to roughly $61,000 rather than using taxable income after deductions. Arizona applies a 2.5% flat rate to $45,842.09, producing $1,146.05." -us,scenario_018,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a projected $15,450 standard deduction rather than $15,750. This overstated taxable income by $300 and tax by $7.50." -us,scenario_018,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used a $15,375 standard deduction rather than $15,750. This overstated taxable income by $375 and tax by $9.38 before rounding." -us,scenario_018,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no state-income-tax value or explanation, so the required output was missing." -us,scenario_018,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used a $15,000 Arizona standard deduction rather than $15,750. That overstated taxable income by $750 and tax by $18.75." -us,scenario_018,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model submitted zero without applying Arizona's 2.5% tax rate. The traced AGI and standard deduction leave $45,842.09 of taxable income, so the liability before refundable credits is $1,146.05." -us,scenario_018,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the stated $68,005 gross wages as already reduced by the separate $6,174 traditional 401(k) contribution. The benchmark defines gross wages before that contribution, so AGI is $61,592.09; the model also used $14,600 rather than the $15,750 Arizona standard deduction." +us,scenario_018,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived $45,842 of taxable income and about $1,146 of tax, but submitted $1,503 instead. Its additional discussion of deducting auto-loan interest also did not support the submitted value and improperly displaced the applicable Arizona standard-deduction calculation." +us,scenario_018,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model applied obsolete progressive Arizona brackets instead of the 2.5% flat rate and used a $14,600 standard deduction rather than $15,750. It then submitted $2,175, a figure unsupported by either its bracket calculation or its medical-expense discussion." +us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model estimated the Arizona standard deduction as $15,000 instead of $15,750. The correct deduction leaves $45,842.09 taxable, producing $1,146.05 rather than $1,165." +us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's stated AGI, deduction, taxable income, and 2.5% rate produced about $1,175, but it submitted $1,471 without any supporting computation. It also used a $14,600 deduction instead of the applicable $15,750." +us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model calculated about $1,137 from its estimated deduction but submitted $1,481. Neither figure follows the correct $15,750 standard deduction and $45,842.09 taxable income." +us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a projected $14,600 Arizona standard deduction instead of $15,750. This overstated taxable income by $1,150 and tax by $28.75." +us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's own AGI, estimated deduction, and 2.5% rate yielded about $1,175, yet it replaced that result with $1,450 based on unsupported premium adjustments. The actual standard deduction is $15,750, and no such adjustment raises the liability." +us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI as $52,592 and used an erroneous $7,700 standard deduction. Federal AGI is $61,592.09 and the Arizona standard deduction is $15,750, leaving $45,842.09 taxable." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used a projected $14,600 standard deduction rather than Arizona's $15,750 deduction for 2026. The extra $1,150 of taxable income caused a $28.75 overstatement." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $1,400 does not apply the $15,750 standard deduction to AGI of $61,592.09. That calculation produces $45,842.09 of taxable income and $1,146.05 at 2.5%." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model estimated the standard deduction as $13,790 instead of $15,750. This overstated taxable income by $1,960 and tax by $49." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI by $9,000, using $52,592 instead of $61,592.09, and also used a $14,600 deduction instead of $15,750. The correct taxable income is $45,842.09, not $37,992." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used a $15,700 standard deduction, $50 below the applicable $15,750. That left taxable income and tax overstated by $50 and $1.25, respectively." +us,scenario_018,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model made a decimal-place transcription error, submitting $11,548 after computing $46,192 × 2.5%, which equals $1,154.80. It also used a $15,400 deduction instead of $15,750." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The submitted $1,941 does not reflect the traced calculation of $61,592.09 AGI less the $15,750 standard deduction. Applying 2.5% to the resulting $45,842.09 yields $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model zeroed the liability despite positive taxable income and no nonrefundable credit eliminating the tax. Arizona taxes $45,842.09 at 2.5%, producing $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used $18,500 of medical and insurance expenses as an itemized deduction instead of comparing the allowable $10,786.71 itemized amount with the $15,750 standard deduction. The larger standard deduction applies, leaving $45,842.09 taxable rather than $43,092." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used a $16,100 standard deduction instead of $15,750. This understated taxable income by $350 and tax by $8.75." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted a $16,100 standard deduction rather than the applicable $15,750. The correct taxable income is $45,842.09 and the tax is $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $45,492 taxable-income estimate embeds a $16,100 deduction rather than $15,750. That $350 understatement reduced tax by $8.75." +us,scenario_018,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model treated Arizona as having progressive rates and applied an approximate average rate to roughly $61,000. Arizona instead applies a 2.5% flat rate to $45,842.09 after the $15,750 standard deduction." +us,scenario_018,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a projected $15,450 standard deduction instead of $15,750. This overstated taxable income by $300 and tax by $7.50." +us,scenario_018,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used a $15,375 standard deduction rather than $15,750. This overstated taxable income by $375 and tax by $9.375 before rounding." +us,scenario_018,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted both an estimated $16,100 standard deduction and an unsupported $2,300 exemption. The applicable calculation subtracts only the selected $15,750 standard deduction from $61,592.09 AGI." +us,scenario_018,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested quantity. +us,scenario_018,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model estimated the standard deduction as $15,000 instead of $15,750. The correct deduction reduces taxable income by another $750 and tax by $18.75." +us,scenario_018,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The zero answer omits Arizona's 2.5% tax on the household's positive taxable income. AGI of $61,592.09 less the $15,750 standard deduction leaves $45,842.09 and $1,146.05 of tax." +us,scenario_018,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $6,174 traditional 401(k) contribution and $289 deductible traditional IRA contribution when deriving AGI, treating $68,055 as AGI. The correct AGI is $61,592.09; after the $15,750 deduction, taxable income is $45,842.09." +us,scenario_018,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model used an incorrect Arizona rate structure: $832 is not 2.5% of its own $45,486 taxable-income estimate. It also used a $14,508 deduction instead of $15,750; the correct flat-rate calculation is 2.5% of $45,842.09." us,scenario_018,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model limited itemized deductions to $13,834 and therefore selected the $16,100 standard deduction. PolicyEngine derives $17,090.34 of itemized deductions, so taxable income is $283,059.84 rather than $284,050." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $138,375 of unreimbursed employee business expenses from AGI and then produced a tax amount inconsistent with its own stated $146,775 taxable income. Those employee expenses do not reduce AGI here; the applicable deductions instead produce $283,059.84 of taxable income." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model double-counted the $42,857 overtime premium by adding it to the already annualized $300,000 gross-wage total. It consequently overstated AGI and taxable income, while also overlooking the $17,090.34 itemized deduction used in the trace." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model denied the explicitly computed $5,708.82 alimony deduction and invented $23,855 of mortgage interest from the mortgage balance despite the instruction that unlisted amounts are zero. The actual deduction sequence gives AGI of $300,150.19 and itemized deductions of $17,090.34." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model replaced the deduction calculation with an unsupported $30,000–$40,000 estimate and never derived taxable income under the applicable rules. The trace uses exactly $17,090.34 of itemized deductions and $283,059.84 of taxable income." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model disallowed the $5,708.82 alimony deduction and applied an obsolete $10,000 SALT treatment, leading it to choose an estimated standard deduction. PolicyEngine instead derives $300,150.19 of AGI and $17,090.34 of itemized deductions." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model capped the relevant itemized deductions at roughly $10,196 and selected a $15,000 standard deduction rather than the $17,090.34 itemized amount. It also added the $900 Additional Medicare Tax to this income-tax output even though that amount belongs in payroll tax." -us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted employer-sponsored insurance premiums from wages, allowed $132,426 of unreimbursed employee expenses, added a personal exemption, and used reverted pre-TCJA brackets. None of those steps belongs in this derivation; taxable income is $283,059.84 under the applicable 2026 rules." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,389 insurance premium from reported wages and built an AMT calculation around disallowed employee-business-expense and personal-exemption assumptions. The regular-tax derivation instead starts from $305,859 of gross income and reaches $283,059.84 of taxable income after alimony and itemized deductions." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used $305,859 as AGI, omitting the $5,708.82 above-the-line alimony deduction, and described an internally confused combination of standard and itemized deductions. The correct sequence uses AGI of $300,150.19 and $17,090.34 of itemized deductions." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model assumed expiration-based pre-TCJA rules, subtracted ESI premiums from wages, and allowed unreimbursed employee expenses subject to a 2% floor. The applicable computation does not use that miscellaneous deduction pathway and instead yields $283,059.84 of taxable income." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced income by both ESI premiums and alimony and then claimed about $146,570 of itemized deductions, driven by the unreimbursed employee expenses. The trace permits neither that wage reduction nor that employee-expense deduction and uses only $17,090.34 of itemized deductions." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted ESI premiums from wages, deducted unreimbursed employee expenses above a 2% floor, restored a personal exemption, and used uncapped property taxes. Those pre-TCJA-style steps incorrectly reduce taxable income; the applicable taxable-income amount is $283,059.84." -us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated the unreimbursed employee expenses as a restored miscellaneous itemized deduction and also restored a personal exemption and pre-TCJA brackets. PolicyEngine uses $17,090.34 of itemized deductions, no such employee-expense deduction, and the applicable 2026 rate schedule." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The zero answer treats the listed expenses as sufficient to eliminate taxable income, principally by counting the $138,375 of unreimbursed employee business expenses and an inferred mortgage-interest deduction. The allowable deduction calculation leaves $283,059.84 taxable, so substantial federal income tax remains." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The $31,472 estimate is consistent with deducting the large unreimbursed employee business expense or otherwise applying reverted pre-TCJA deductions. The applicable deduction sequence leaves $283,059.84 of taxable income and produces $68,056.71 under the 2026 rates." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model correctly derived AGI but selected the $16,100 standard deduction after overlooking the full $17,090.34 itemized deduction. That $990.34 deduction difference lowers taxable income from its $284,050 estimate to $283,059.84." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model says it adjusted reported wages for employer-sponsored insurance and then used the standard deduction. Reported gross wages are not reduced again by that premium, and the filer uses $17,090.34 of itemized deductions rather than the standard deduction." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model selected the standard deduction and separately applied a $196 nonitemizer charitable deduction. The trace instead itemizes $17,090.34 in total, producing $283,059.84 of taxable income before applying the rate schedule." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model explicitly used the standard deduction and omitted the larger $17,090.34 itemized deduction. This overstated taxable income relative to the traced $283,059.84 amount." -us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used roughly $306,000 of AGI and a standard deduction, omitting the $5,708.82 alimony adjustment and the larger itemized deduction. The correct inputs are $300,150.19 of AGI and $17,090.34 of itemized deductions." -us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model deducted $132,372 of unreimbursed employee expenses, restored a personal exemption, and applied reverted pre-TCJA brackets. Those steps incorrectly reduce taxable income to about $148,321 instead of the traced $283,059.84." -us,scenario_020,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model reduced reported wages by ESI premiums and allowed the unreimbursed employee expenses under a 2% floor using pre-TCJA rules. The applicable computation retains the $300,000 wage input and uses $17,090.34 of total itemized deductions." -us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model limited itemized deductions to the $13,834 property-tax amount, selected the $16,100 standard deduction, and then added a separate $196 nonitemizer charitable deduction. The trace instead uses $17,090.34 of itemized deductions, yielding $283,059.84 of taxable income." -us,scenario_020,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The unexplained zero treats deductions or credits as eliminating all liability. The traced deductions leave $283,059.84 of taxable income and no listed nonrefundable credit offsets the resulting tax." -us,scenario_020,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted the $42,857 overtime premium by adding it to the $300,000 annual gross-wage total and omitted the $5,708.82 alimony deduction. Its submitted $68,513.50 also contradicts its own stated $85,241.65 calculation; the correct deduction sequence produces $283,059.84 of taxable income." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model limited itemized deductions to the listed $13,834 of real-estate tax and therefore selected the $16,100 standard deduction. The computation instead uses $17,090.34 of itemized deductions, lowering taxable income from the model's $284,050 to $283,059.84." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model deducted $138,375 of unreimbursed employee business expenses from AGI even though that amount is not an above-the-line deduction in this computation. It then contradicted its own bracket calculation by changing an estimated $27,268 liability on $146,775 of taxable income into $66,661 without a valid tax step." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model double-counted the $42,857 FLSA overtime premium by adding it to the already annualized $300,000 of gross wages. It consequently started from $348,716 rather than $305,859 and never followed the traced $300,150.19 AGI and $283,059.84 taxable-income calculation." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model rejected the traced $5,708.82 alimony deduction and invented $23,855 of mortgage interest from the mortgage balance despite the instruction that unlisted expenses are zero. Those two errors replaced the actual $17,090.34 itemized deduction with a fabricated $34,051 deduction." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model guessed that itemized deductions were $30,000–$40,000 instead of calculating the traced $17,090.34 amount. Its $72,700 output does not follow from its own asserted taxable-income range or any completed bracket calculation." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model disallowed the $5,708.82 alimony deduction and selected an estimated $15,400 standard deduction after imposing a $10,000 SALT cap. The traced computation instead has AGI of $300,150.19, itemized deductions of $17,090.34, and taxable income of $283,059.84." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model used a $15,000 standard deduction rather than the traced $17,090.34 itemized deduction. It also added the $900 Additional Medicare Tax to this income-tax output even though that amount belongs in employee payroll tax, then submitted $76,500 despite deriving approximately $70,467." +us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted employer-sponsored insurance premiums from wages, restored a personal exemption, and deducted $132,426 of unreimbursed employee expenses under obsolete pre-TCJA rules. The computation instead starts with the full $300,000 wage input and reaches $283,059.84 of taxable income using only the traced alimony and itemized deductions." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted $8,389 of employer insurance premiums from the stated gross wages and applied restored employee-expense deductions and a personal exemption. Its AMT calculation was therefore built on an incorrect regular-tax and AMTI base rather than the traced $300,150.19 AGI and $283,059.84 taxable income." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used $305,859 as AGI, omitting the $5,708.82 alimony deduction, and described mutually incompatible use of both standard and itemized deductions. The traced sequence uses $300,150.19 AGI and $17,090.34 of itemized deductions." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model assumed expiration of the governing rules, subtracted ESI premiums from gross wages, and restored the miscellaneous itemized deduction for unreimbursed employee expenses. That obsolete pathway generated an artificial AMT calculation instead of the traced regular-tax computation on $283,059.84." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced income by both ESI premiums and alimony to obtain $291,761 and then claimed $146,570 of itemized deductions, principally from unreimbursed employee expenses. The traced amounts are $300,150.19 of AGI and $17,090.34 of itemized deductions." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted ESI premiums from wages and restored full property-tax treatment, miscellaneous employee-expense deductions, and a personal exemption. These obsolete deductions reduced taxable income by over $130,000 relative to the traced $283,059.84." +us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated the $138,375 of employee expenses as a revived miscellaneous itemized deduction and added a personal exemption under reverted pre-TCJA law. The actual itemized deduction is $17,090.34, not $146,402, and taxable income is $283,059.84." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the mortgage balance as mortgage interest and the large unreimbursed employee expenses as deductible enough to erase taxable income. No mortgage-interest payment was listed, and the traced deductions leave $283,059.84 of taxable income rather than zero." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The $31,472 answer implies that the model substantially deducted the $138,375 employee-expense input or used a comparable obsolete deduction shortcut. The traced calculation allows $17,090.34 of itemized deductions and yields $283,059.84 of taxable income before applying the tax calculation." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model selected the $16,100 standard deduction because it failed to compute the traced $17,090.34 itemized deduction. That left taxable income $990.16 too high at $284,050 instead of $283,059.84." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model subtracted employer-sponsored insurance premiums as an additional income adjustment even though the stated $300,000 gross-wage input is the income base used by the computation. It also used the standard deduction instead of the traced $17,090.34 itemized deduction." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used the standard deduction plus a separate $196 nonitemizer charitable deduction rather than the traced $17,090.34 itemized deduction. This produced the wrong taxable-income base and omitted the traced NIIT and capital-gains-tax computation steps from the stated derivation." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model selected the 2026 standard deduction and omitted the $5,708.82 alimony adjustment from its stated derivation. The computation instead itemizes $17,090.34 after reducing AGI to $300,150.19." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used an undifferentiated $306,000 AGI and standard-deduction shortcut. The traced computation deducts $5,708.82 of alimony, itemizes $17,090.34, and taxes $283,059.84." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model restored the miscellaneous deduction for employee expenses and a personal exemption, reducing taxable income to $148,321. Those deductions do not enter the traced computation, which itemizes $17,090.34 and leaves $283,059.84 taxable." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model subtracted $8,389 of ESI premiums from the stated gross wages and applied a $132,426 employee-expense deduction under reverted pre-TCJA rules. The computation instead uses the full wage input, the alimony adjustment, and $17,090.34 of itemized deductions." +us,scenario_020,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted ESI premiums from the stated gross wages and treated the $196 donation as an above-the-line deduction while omitting the $5,708.82 alimony deduction. The traced AGI is $300,150.19 and the donation is part of the itemized-deduction calculation that totals $17,090.34." +us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so the required structured answer was missing." +us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model treated $13,834 as the entire itemized amount, chose the standard deduction, and then added a $196 nonitemizer charitable deduction. The computation instead itemizes $17,090.34 and produces taxable income of $283,059.84, not $283,854." +us,scenario_020,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer ignores the $283,059.84 of taxable income remaining after the traced alimony and itemized deductions. With no reasoning supplied, the output is consistent with incorrectly treating the large employee-expense or mortgage-balance inputs as deductions that eliminate taxable income." +us,scenario_020,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted the $42,857 overtime premium by adding it to the $300,000 annual gross-wage total and omitted the $5,708.82 alimony deduction. Its explanation calculates $85,241.65 from that inflated base but then submits the unrelated value $68,513.50." +us,scenario_020,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,The model applied married-filing-jointly-style brackets to a single filer and invented both a qualified-business-income deduction and a child tax credit despite no business income or child being listed. The traced computation uses single-filer taxation with no such deduction or credit. us,scenario_020,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_020,payroll_tax,claude-fable-5,llm_error,other,False,"The model explicitly derived the correct $11,439, $4,350, and $900 components and their $16,689 sum, then submitted $15,200, a number unsupported by either its gross-wage calculation or its alternative ESI-adjusted calculation." -us,scenario_020,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model double-counted the $42,857 overtime premium even though the prompt states that gross wages already include overtime, used an outdated $168,600 Social Security wage base, and then submitted $23,119.50 despite none of its displayed calculations producing that amount." -us,scenario_020,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model explicitly calculated the correct $16,689 total using the $184,500 wage base, then discarded it for unrelated wage-base estimates and submitted $13,283.40, which does not follow from any calculation it showed." -us,scenario_020,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model's own components produced $16,168.20 using its mistaken wage base, but it submitted $6,219. The submitted value omits or corrupts major FICA components and has no support in its reasoning." -us,scenario_020,payroll_tax,claude-opus-5,llm_error,other,False,"The model derived the exact reference components and $16,689 total, then replaced that result with an unsupported $15,866 adjustment for alleged wage-base uncertainty." -us,scenario_020,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model carried the 2025 Social Security wage base of $176,100 into 2026. The 2026 base used in the computation is $184,500, raising employee Social Security tax from $10,918.20 to $11,439 while leaving the $4,350 Medicare and $900 surtax components unchanged." -us,scenario_020,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"The model first derived $16,689 with the correct $184,500 Social Security wage base, then substituted an erroneous $182,600 base. That substitution understated Social Security tax by $117.80 and produced the wrong total." -us,scenario_020,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $183,700 Social Security wage base instead of $184,500. This understated the capped Social Security component by $49.60, represented as $50 after its rounding, while its Medicare components were otherwise aligned." -us,scenario_020,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly deducted the $8,389 employer-sponsored insurance premium from the $300,000 payroll-tax wage input and also used the 2025 Social Security cap of $176,100. The trace taxes $300,000 for Medicare and Additional Medicare Tax and applies the 2026 $184,500 Social Security wage base." -us,scenario_020,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The stated formula requires $11,439 of capped Social Security tax, $4,350 of Medicare tax, and $900 of Additional Medicare Tax, totaling $16,689. Its unexplained $15,516 submission therefore miscomputed at least one payroll-tax component despite naming all three." -us,scenario_020,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced Medicare-taxable wages by the ESI premium instead of using the $300,000 wage input from the trace. It also failed to produce the $11,439 Social Security component needed with the 2026 wage base, yielding an understated total." -us,scenario_020,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly reduced Medicare and Additional Medicare Tax wages to $291,611 for the ESI premium and used an understated Social Security cap that produced only about $10,993. The required bases produce $11,439, $4,350, and $900." -us,scenario_020,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model deducted the $8,389 ESI premium from Medicare wages and used a $181,800 Social Security cap. The trace instead uses $300,000 for both Medicare calculations and the 2026 $184,500 Social Security wage base." -us,scenario_020,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"The model used a round $180,000 projected Social Security wage base rather than the 2026 $184,500 limit. This reduced Social Security tax from $11,439 to $11,160 and understated total payroll tax by $279." -us,scenario_020,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The correct components under the formula it named are $11,439 of Social Security tax, $4,350 of Medicare tax, and $900 of Additional Medicare Tax. Its $18,348 answer overstates their sum by $1,659 without identifying any Texas employee payroll tax or other included component." -us,scenario_020,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model incorrectly said Additional Medicare Tax was not triggered even though this single filer's $300,000 wages exceed the $200,000 threshold by $100,000, generating $900. Its $11,689 total also fails to include the full $11,439 Social Security and $4,350 regular Medicare components." -us,scenario_020,payroll_tax,grok-4.3,llm_error,other,False,"The model used the wrong $176,100 Social Security wage base, but even its stated components sum to $16,168.20, not $10,578. The submitted value is an arithmetic or transcription failure that drops $5,590.20 from its own calculation." -us,scenario_020,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"The model used an estimated $183,144 Social Security wage base instead of $184,500. This reduced the Social Security component from $11,439 to about $11,355 and understated the total by $84." -us,scenario_020,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced Medicare and Additional Medicare Tax wages to $291,611 for the health premium and used a projected $184,000 Social Security base. The trace uses $300,000 for both Medicare components and $184,500 for Social Security." -us,scenario_020,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required structured-output contract." -us,scenario_020,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated the household as having no employee payroll-tax liability despite $300,000 of wages. Those wages generate $11,439 of Social Security tax, $4,350 of Medicare tax, and $900 of Additional Medicare Tax." -us,scenario_020,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model double-counted the $42,857 overtime premium even though the prompt says overtime is included in the $300,000 gross-wage total, and it used the 2025 Social Security base of $176,100. It then submitted $19,436.31 although its own erroneous component calculation totaled $17,175.34, adding a further unsupported $2,260.97." -us,scenario_020,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model correctly identified that Texas has no state income tax, then overrode that rule by inserting an unsupported $2,380 nonrefundable state-tax amount. It treated a nonexistent modeled nonrefundable state liability as part of the Texas state income tax calculation instead of leaving the pre-refundable state income tax at $0." +us,scenario_020,payroll_tax,claude-fable-5,llm_error,other,False,"The model explicitly derived the correct $16,689 total, then substituted an unexplained $15,200 final value that matches none of its calculations. Its alternative ESI-premium treatment also incorrectly reduced FICA wages despite the prompt defining $300,000 as the annual gross-wage total used here." +us,scenario_020,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model repeatedly added the $42,857 overtime premium to the $300,000 gross-wage total even though the prompt says gross wages already include overtime, and it used outdated wage-base figures. Its submitted $23,119.50 also matches none of its stated totals, including the correct treatment it briefly calculated." +us,scenario_020,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model correctly computed $11,439 of Social Security tax, $4,350 of Medicare tax, and $900 of Additional Medicare Tax, but discarded their $16,689 sum and submitted an unexplained $13,283.40. That final value follows no computation in its reasoning." +us,scenario_020,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model's stated components sum to $16,168.20 under its outdated wage-base assumption, yet it submitted $6,219. It omitted or arbitrarily displaced major payroll-tax components when producing the final value." +us,scenario_020,payroll_tax,claude-opus-5,llm_error,other,False,"The model derived the exact $16,689 result from the correct three components, then replaced it with $15,866 solely for unspecified wage-base uncertainty. The fixed 2026 wage base in the benchmark must be applied rather than adjusting a correctly computed result." +us,scenario_020,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model reused the 2025 Social Security wage base of $176,100 for 2026. Applying the 2026 $184,500 cap produces $11,439 of Social Security tax and raises the total from $16,168.20 to $16,689." +us,scenario_020,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model first used the correct $184,500 Social Security wage base and obtained $16,689, then replaced it with an approximate $182,600 base. The benchmark requires the 2026 cap of $184,500, not a revised estimate." +us,scenario_020,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model used an estimated $183,700 Social Security wage base instead of the 2026 $184,500 limit. This understated Social Security tax by $50 and produced $16,639 rather than $16,689." +us,scenario_020,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model used the outdated $176,100 Social Security cap and subtracted $8,389 of ESI premiums from Medicare and Additional Medicare wages. The trace taxes the stated $300,000 gross wages and uses the $184,500 cap, yielding components of $11,439, $4,350, and $900." +us,scenario_020,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The submitted $15,516 is inconsistent with applying the stated rates to $300,000 and the 2026 $184,500 Social Security cap. The correct three-component calculation is $11,439 plus $4,350 plus $900, so the model used an incorrect capped wage base or omitted part of the Medicare surtax computation." +us,scenario_020,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model explicitly reduced Medicare-taxable wages by the $8,389 ESI premium, while the trace applies Medicare taxes to the stated $300,000 gross wages. It also failed to use the exact $184,500 Social Security cap needed for the $11,439 component." +us,scenario_020,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model subtracted ESI premiums to obtain $291,611 of Medicare wages and used an incorrect Social Security cap producing only about $10,993 of Social Security tax. Medicare and Additional Medicare taxes instead apply to $300,000 here, and Social Security tax is $11,439." +us,scenario_020,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model used a $181,800 Social Security cap rather than $184,500 and reduced Medicare wages from $300,000 to $291,611 for ESI premiums. Those two base errors understated all three reported components relative to $11,439, $4,350, and $900." +us,scenario_020,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model imposed a projected $180,000 Social Security wage base rather than the 2026 $184,500 limit. This reduced Social Security tax from $11,439 to $11,160 and understated the total by $279." +us,scenario_020,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The submitted $18,348 exceeds the correct $16,689 despite correctly identifying the three applicable federal components and no Texas employee payroll tax. Its number implies that it failed to cap Social Security wages at the 2026 $184,500 limit or otherwise overstated a component." +us,scenario_020,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model explicitly omitted Additional Medicare Tax even though a single filer's $300,000 of wages exceeds the $200,000 threshold by $100,000. It also failed to produce the correct ordinary Social Security and Medicare sum of $15,789 before adding the $900 surtax." +us,scenario_020,payroll_tax,grok-4.3,llm_error,other,False,"The model used the outdated $176,100 Social Security cap, and its stated components actually sum to $16,168.20 rather than $10,578. The submitted value therefore contains both a wage-base error and a basic aggregation error." +us,scenario_020,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model used an estimated Social Security wage base of about $183,144 instead of the 2026 $184,500 cap. That lowered Social Security tax from $11,439 to $11,355 and understated the total by $84." +us,scenario_020,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model subtracted the $8,389 health premium from FICA wages and used an approximate $184,000 Social Security cap. The trace uses $300,000 for Medicare and Additional Medicare taxes and the exact $184,500 Social Security cap." +us,scenario_020,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model correctly used the $184,500 Social Security cap but incorrectly reduced Medicare and Additional Medicare wages to $291,611 by subtracting ESI premiums. Using the stated $300,000 gross wages makes those components $4,350 and $900, bringing the total to $16,689." +us,scenario_020,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no payroll-tax value or explanation. It therefore failed the required output contract rather than supplying a substantive estimate. +us,scenario_020,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model assigned zero payroll tax despite $300,000 of employee wages. Those wages generate $11,439 of Social Security tax, $4,350 of Medicare tax, and $900 of Additional Medicare Tax." +us,scenario_020,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model double-counted the $42,857 overtime premium by adding it to gross wages even though the prompt states that gross wages already include overtime, and it used the outdated $176,100 Social Security cap. Its submitted $19,436.31 also does not match its own erroneous component sum of $17,175.34." +us,scenario_020,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model treated overtime premiums as additional to the stated $300,000 gross wages and submitted $20,303 without a component calculation. Gross wages already include overtime, and capping Social Security wages at $184,500 while applying Medicare taxes to $300,000 yields $16,689." +us,scenario_020,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model correctly identified that Texas has no state individual income tax but then inserted an unexplained $2,380 nonrefundable state-tax amount. No state or local jurisdictional rule applies to create that liability for this Texas household, so the state-income-tax computation yields $0." us,scenario_021,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_021,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented qualification through Missouri's aged/disabled Medicaid pathway even though PolicyEngine assigned the head no Medicaid category. It treated age, disability, and assets of $5,900 as sufficient after noting an ABD income limit around 85% FPL, while the engine's rules leave the head in category NONE and return not Medicaid eligible." -us,scenario_021,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly applied ABD disregards and medical-expense treatment to create Missouri Medicaid eligibility. Under the engine trace, the head receives no SSI, has no qualifying Medicaid category, and does not qualify through any Missouri aged/blind/disabled pathway despite age and disability." -us,scenario_021,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model improperly counted the head as eligible through an aged/blind/disabled or Medicare Savings Program pathway. PolicyEngine's eligibility category is NONE, and the cited age, disability, surviving-spouse status, income, and assets do not satisfy any modeled Missouri Medicaid pathway." -us,scenario_021,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used a shortcut that equates being 85 with low countable income and modest assets to Medicaid eligibility. The correct engine derivation gives MAGI income of 1.72 x FPL, SSI receipt of zero, no Medicaid category, and therefore no Medicaid eligibility." -us,scenario_021,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model treated the $22,000 Social Security retirement benefit as payroll-taxable earnings even though Social Security benefits are not wages and are outside the employee FICA base. It also applied the combined employer-plus-employee 15.3% Social Security and Medicare rate rather than the employee-side payroll tax definition, producing a positive tax despite zero wage or self-employment income." +us,scenario_021,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model asserted aged/disabled eligibility after acknowledging roughly $27,508 of income, but never applied Missouri's ABD income test to establish qualification. Disability and assets below its assumed resource limit do not create eligibility when the person qualifies through no Medicaid category." +us,scenario_021,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invoked general and medical-expense disregards without calculating countable income or showing that those deductions reduce income below Missouri's ABD limit. Age, disability, and resources do not substitute for satisfying the pathway's income requirement, and the resulting Medicaid category is NONE." +us,scenario_021,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated Medicaid eligibility with an ABD spend-down or Medicare Savings Program pathway and incorrectly described $27,530 as low enough without applying a qualifying limit. It also treated Social Security as mostly excluded and assumed a favorable resource rule, neither of which establishes any PolicyEngine Medicaid category." +us,scenario_021,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model labeled the head's income very low without applying a Missouri Medicaid eligibility pathway or threshold. The engine derives MAGI income of 1.72 times FPL and no qualifying category, so age and modest assets alone yield no Medicaid eligibility." +us,scenario_021,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model inferred eligibility directly from age and vaguely characterized income and resources as limited, without testing a specific Missouri category. The head qualifies through none of the Medicaid pathways, so those characteristics do not produce eligibility." +us,scenario_021,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model incorrectly treated $22,000 of Social Security retirement benefits as covered wages subject to payroll tax. It also applied the combined employer-and-employee Social Security and Medicare rates of 12.4% and 2.9%, even though the requested output includes only employee-side taxes; the listed household has no payroll-taxable earnings, so the tax is $0." us,scenario_021,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_021,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_021,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model treated elderly/disabled status as eliminating the relevant SNAP income limit and proceeded to calculate an allotment from net income, instead of applying Missouri's income eligibility screen that disqualifies this household at $27,508 of annual income. Its calculation also used an inflated one-person maximum SNAP allotment of about $827 per month, producing a positive benefit where the correct eligibility step yields value = 0." -us,scenario_021,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not return a parseable value for state_income_tax_before_refundable_credits, so it failed the required output contract rather than making a substantive tax calculation." -us,scenario_021,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model taxed the $5,508 taxable private pension directly at an approximate Missouri rate and skipped Missouri’s taxable-income calculation and threshold structure. After applying the Missouri state income-tax computation for this elderly single filer, the income falls below the taxable threshold, so the state income tax before refundable credits is $0." +us,scenario_021,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly identified the elderly/disabled net-income test and calculated about $2,063 of monthly net income, but never compared that amount with Missouri’s one-person SNAP net-income limit. It instead proceeded to the benefit formula and also used an erroneous one-person maximum allotment of about $827 per month; failing the net-income test yields $0." +us,scenario_021,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value for the requested output and therefore failed the required submission contract. +us,scenario_021,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $5,508 taxable pension as Missouri taxable income, ignored the applicable deductions and 2026 taxable-income threshold, and applied an unsupported flat 1.9% rate directly to that pension. Missouri taxable income is reduced to zero before the rate calculation, yielding $0 of tax before refundable credits." us,scenario_021,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model counted only the $8,829 real-estate tax as SALT and used the full $15,393 stated donations, producing itemized deductions of $24,222 instead of $29,654.81. The applicable itemized amounts are $14,807.14 of SALT and $14,847.67 of charitable deductions, which reduce taxable income to $74,633.30." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $8,351 of unreimbursed employee business expenses above the line, treated all Social Security as taxable, selected head-of-household status, and used a standard deduction instead of the optimal itemized deductions. It also stated bracket tax of $10,564 but submitted $11,316, so its final value does not follow its own arithmetic." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model treated the taxpayer as a qualifying surviving spouse and applied joint brackets and a joint standard deduction. With no qualifying child, the taxpayer uses single status, itemizes $29,654.81, and also deducts $910 of QBI and $3,951.05 under the senior deduction." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented $12,771 of mortgage interest from the outstanding mortgage balance and allowed the full $6,000 senior deduction despite its income phaseout. It also applied qualifying-surviving-spouse brackets; the actual senior deduction is $3,951.05 and no unlisted mortgage-interest payment is inferred." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Although the model began with single status, its claimed $8,547 tax is incompatible with the single tax schedule it described and follows neither its stated taxable income nor valid bracket arithmetic. It also understated itemized deductions by omitting the income-tax component of the $14,807.14 SALT deduction." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the loan balance, restored unreimbursed employee expenses and personal exemptions under an erroneous TCJA-sunset assumption, and applied joint rates. The operative 2026 law instead includes the phased $3,951.05 senior deduction, retains the relevant current-law bracket structure, and yields taxable income of $74,633.30." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used qualifying-surviving-spouse status, a joint standard deduction, and joint brackets even though no qualifying child was listed. It also omitted the separate phased senior deduction and failed to use the optimal $29,654.81 itemized deduction." -us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly restored a personal exemption and the miscellaneous itemized deduction for unreimbursed employee expenses. It also omitted the $910 QBI deduction and $3,951.05 senior deduction and used obsolete 10%/15%/25% brackets rather than the operative 2026 schedule." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model deducted unreimbursed employee business expenses and a personal exemption under an obsolete post-sunset regime. It omitted the phased $3,951.05 senior deduction and therefore used both the wrong taxable-income calculation and the wrong bracket schedule." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's generic explanation does not identify the deductions used and invokes mortgage interest even though no mortgage-interest payment was listed. The required derivation subtracts $29,654.81 of itemized deductions, $910 of QBI, and a $3,951.05 senior deduction from AGI, producing $74,633.30 of taxable income." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied a pre-TCJA itemized-deduction and personal-exemption regime. The 2026 computation instead uses current itemized deductions of $29,654.81, no personal exemption, a $910 QBI deduction, and the phased $3,951.05 senior deduction." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted unreimbursed employee business expenses and a personal exemption under a supposed TCJA expiration. It also omitted the $910 QBI deduction and $3,951.05 senior deduction and applied obsolete tax rates." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model omitted the $3,951.05 senior deduction and understated itemized deductions by using $25,393 rather than $29,654.81. Those omissions inflated its taxable income from $74,633.30 to $82,846." -us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the phased $3,951.05 senior deduction and understated SALT by excluding the income-tax component, using itemized deductions of only $24,222. These errors raised taxable income from $74,633.30 to $84,016.75." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model disregarded $52,635 of taxable wages and asserted that deductions or unspecified offsets eliminated all tax. The listed income produces AGI of $109,149.16, and the allowed deductions leave $74,633.30 taxable rather than zero." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The submitted $1,237 implies that the model either removed most taxable income or subtracted nonexistent nonrefundable credits. No applicable nonrefundable credit offsets the bracket tax on $74,633.30 of taxable income, which is $11,131.33." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model applied surviving-spouse joint brackets and subtracted a personal exemption. The taxpayer instead uses single brackets, has no personal exemption, and receives the separate phased senior deduction of $3,951.05." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The answer falls below the tax generated by the traced taxable income and provides no figures supporting its deduction calculation. The specified deductions total $34,515.86 and leave taxable income of $74,633.30; applying the 2026 brackets yields $11,131.33." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model used surviving-spouse joint-rate treatment and an age-based standard deduction. With no qualifying child, single rates apply, and the taxpayer itemizes $29,654.81 while separately deducting $910 of QBI and $3,951.05 under the senior deduction." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model applied the surviving-spouse tax schedule rather than the single-filer schedule required in the absence of a qualifying child. Its explanation also omits the $910 QBI deduction and the phased $3,951.05 senior deduction from the taxable-income derivation." -us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model used qualifying-surviving-spouse status, a $30,000 standard deduction, and joint brackets. The taxpayer must use single status and optimally itemizes $29,654.81, with separate QBI and phased senior deductions reducing taxable income to $74,633.30." -us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly restored unreimbursed employee expenses and a personal exemption under a post-TCJA-sunset calculation. It omitted the $3,951.05 senior deduction and $910 QBI deduction and therefore derived the wrong taxable income." -us,scenario_022,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model taxed all $20,735 of Social Security instead of the taxable $17,624.75 and improperly deducted unreimbursed employee expenses under a sunset regime. It also omitted the QBI and senior deductions and applied obsolete 10%/15%/25% brackets." -us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model omitted the $910 QBI deduction and used an unsupported $2,700 California income-tax amount, producing SALT and total itemized deductions different from the traced $14,807.14 and $29,654.81. These errors left taxable income $3,642.37 too high." -us,scenario_022,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model asserted that the standard deduction and unspecified credits eliminated tax despite AGI of $109,149.16. No listed nonrefundable credits erase the liability, and the optimal deductions leave $74,633.30 subject to tax." -us,scenario_022,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted taxable Social Security when summing AGI, reporting $126,773.50 instead of $109,149.16, and invented $12,771 of mortgage interest from the mortgage balance. It also applied qualifying-surviving-spouse brackets and omitted the $910 QBI and phased $3,951.05 senior deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It included only the listed $8,829 of real-estate tax in SALT and omitted the computed California income tax that raises the SALT deduction to $14,807.14. That understated itemized deductions and produced taxable income of about $80,066 instead of $74,633.30." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It treated all Social Security as gross income, improperly deducted unreimbursed employee expenses above the line, used head-of-household status, and selected an estimated standard deduction. The required path uses $17,624.75 of taxable Social Security and the $34,515.86 combined itemized, QBI, and senior deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It treated the taxpayer as a qualifying surviving spouse and applied the joint standard deduction and joint brackets despite the absence of a qualifying child. The calculation instead uses the single schedule and itemizes $29,654.81 before the separate QBI and senior deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It invented $12,771 of mortgage interest from the mortgage balance even though unlisted expenses are zero, granted the full $6,000 senior deduction without its MAGI phaseout, and used qualifying-surviving-spouse brackets. The allowed senior deduction is $3,951.05, and no mortgage-interest deduction enters the trace." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Although it identified the main income, charity, property tax, QBI, and senior-deduction components, it stated a single-schedule tax of only $8,547 on roughly $78,017. That amount does not follow from the 2026 single brackets, and it also omitted computed California income tax from SALT." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It invented mortgage interest, restored unreimbursed employee deductions and personal exemptions under an erroneous TCJA-sunset theory, and used qualifying-surviving-spouse rates. For 2026 the calculation instead uses current 10%/12%/22% single brackets and only the traced $29,654.81 itemized deduction, $910 QBI deduction, and $3,951.05 senior deduction." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It selected an estimated joint standard deduction and joint brackets based solely on the surviving-spouse flag. With no qualifying child, the taxpayer uses the single schedule, and itemizing $29,654.81 plus the separate QBI and phased senior deductions is optimal." +us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied obsolete unreimbursed-employee deductions and a personal exemption while omitting the $910 QBI deduction and $3,951.05 senior deduction. Its itemized total also understated SALT and overstated deductible charity relative to the traced amounts." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It deducted unreimbursed employee business expenses and a personal exemption under an erroneous post-TCJA-expiration calculation. Those deductions do not enter the 2026 trace; the applicable additions beyond itemizing are the $910 QBI deduction and $3,951.05 senior deduction." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its explanation invokes mortgage interest even though no mortgage-interest payment was listed and unlisted expenses are zero. The correct itemized deduction consists of $14,807.14 of SALT and $14,847.67 of charity, followed by the QBI and phased senior deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,It used a pre-TCJA itemized-deduction and personal-exemption framework for 2026. The traced calculation has no personal exemption or unreimbursed-employee deduction and instead subtracts the QBI and phased senior deductions. +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It deducted $6,168 of unreimbursed employee expenses and a $5,050 personal exemption, neither of which belongs in this 2026 calculation. It also omitted the $910 QBI deduction and $3,951.05 senior deduction and understated SALT." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It limited SALT to an assumed $10,000 and omitted the phased $3,951.05 senior deduction. The trace allows $14,807.14 of SALT, producing total itemized deductions of $29,654.81 and taxable income of $74,633.30." +us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It omitted computed California income tax from SALT and omitted the phased senior deduction, leaving taxable income at $84,016.75. It also applied 2025 rather than 2026 brackets; the traced deductions reduce taxable income to $74,633.30 before applying the 2026 schedule." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It incorrectly claimed there was no taxable wage income and that deductions or credits eliminated the liability. Wages alone are $52,635, total AGI is $109,149.16, and the allowed deductions leave $74,633.30 taxable with no nonrefundable credit offset." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Its $1,237 result implies that it subtracted large nonrefundable credits or removed most of the taxable income, but the household has no listed basis for such credits. The traced deductions leave $74,633.30 taxable, and applying the 2026 brackets yields $11,131.33." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse joint brackets and a personal exemption. With no qualifying child the taxpayer follows the single schedule, and the deduction path is $29,654.81 itemized plus $910 QBI and $3,951.05 senior deductions, with no personal exemption." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It named the relevant deduction categories but its $10,247 result does not apply their traced amounts. The deductions total $34,515.86 and leave $74,633.30 taxable, whose 2026 bracket tax is $11,131.33." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It applied surviving-spouse joint-rate treatment and an age-based standard deduction. The taxpayer lacks the qualifying child required for that filing status, and the optimal calculation uses the single brackets with itemized, QBI, and phased senior deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It applied a surviving-spouse tax schedule instead of the single schedule required without a qualifying child. The allowable deductions produce $74,633.30 of taxable income, to which the 2026 single brackets apply." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"It used a qualifying-surviving-spouse standard deduction and joint brackets. It also failed to itemize the traced $29,654.81 and omitted the separate $910 QBI and $3,951.05 senior deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied obsolete unreimbursed-employee and personal-exemption deductions while omitting the QBI and phased senior deductions. Its SALT and charity amounts also differ from the traced $14,807.14 and $14,847.67." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It treated the full $20,735 of Social Security as taxable instead of $17,624.75 and deducted unreimbursed employee expenses under an obsolete rule. It also omitted the QBI and phased senior deductions, overstating taxable income to $79,677." +us,scenario_022,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It invented rental depreciation and a charitable-contribution floor, reducing both AGI and the charity deduction without support from the inputs. It also used an incorrect SALT amount; the trace uses $109,149.16 AGI, $14,807.14 SALT, and $14,847.67 of deductible charity." +us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It supplied no numeric output or explanation for the requested variable, so the required structured answer was missing." +us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used an estimated $2,700 California income tax in SALT instead of the computed amount and omitted the $910 QBI deduction. The trace yields $14,807.14 of total SALT, $29,654.81 of itemized deductions, and $74,633.30 of taxable income." +us,scenario_022,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It asserted that a standard deduction and nonrefundable credits eliminated the tax despite $109,149.16 of AGI and no applicable credit identified. The allowed deductions leave $74,633.30 taxable rather than zero taxable income." +us,scenario_022,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It double-counted taxable Social Security when summing AGI, producing $126,773.50 instead of $109,149.16, and invented $12,771 of mortgage interest from the loan balance. It also used qualifying-surviving-spouse rates and omitted the QBI and phased senior deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It treated itemized deductions as nonrefundable credits, invented mortgage interest, and concluded that deductions made taxable income negative. The traced deductions total $34,515.86, leaving positive taxable income of $74,633.30 and no credit offset." us,scenario_022,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_022,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model treated age 77 plus assumed Medicare enrollment as sufficient to establish an aged Medicaid category. PolicyEngine does not grant Medicaid eligibility from Medicare status alone; this head has medicaid_category NONE, receives no SSI, and fails the income and categorical requirements for California Medicaid." +us,scenario_022,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly treated Medicare enrollment plus age 77 as sufficient for Medicaid eligibility. Medicare status does not establish Medicaid eligibility, and this person receives no SSI and has medicaid_category NONE, so no California Medicaid pathway applies." us,scenario_022,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_022,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model used a 1.2% California SDI rate instead of the 2026 rate of 1.3%, understating the state contribution by $52.64. Its submitted $4,557.61 also contradicts its own stated component sum of $4,658.20." -us,scenario_022,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model omitted California's mandatory employee SDI contribution of $684.26 by asserting that no state payroll tax applies. It also miscomputed 1.45% of $52,635 as $762.71 instead of $763.21 and submitted a number inconsistent with its stated total." -us,scenario_022,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,The model included California SDI but applied a 1.2% rate instead of the 2026 rate of 1.3%. This produced $631.62 rather than $684.26 and understated total payroll tax by $52.63. -us,scenario_022,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model incorrectly excluded California SDI even though the requested output expressly includes mandatory employee state payroll taxes. Adding the $684.26 SDI contribution to its federal FICA components produces $4,710.84 before component-level rounding, matching the traced $4,710.83 total." -us,scenario_022,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model omitted the mandatory California employee SDI contribution of $684.26. It also rounded its stated $4,026.58 federal FICA subtotal down to $4,026.00 without justification." -us,scenario_022,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,The model applied a 1.1% California SDI rate instead of the 2026 rate of 1.3%. That reduced the state component from $684.26 to $578.99 and understated payroll tax by $105.26. -us,scenario_022,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model wrongly treated California SDI as nonmandatory and omitted the $684.26 employee contribution. Its submitted $3,264.37 also discards nearly all of the $763.21 Medicare tax despite correctly deriving a $4,026.58 federal FICA subtotal." -us,scenario_022,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model's $579 California SDI component reflects a 1.1% rate rather than the 2026 rate of 1.3%. The correct state component is $684.26, with the remaining small difference arising from its whole-dollar rounding of the federal components." -us,scenario_022,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a 1.1% California SDI rate instead of 1.3%. The state contribution is $684.26 rather than $578.99, raising total payroll tax to $4,710.83." -us,scenario_022,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,The model calculated only federal Social Security and Medicare taxes and omitted the mandatory California SDI contribution. The missing state component is $684.26. -us,scenario_022,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a 1.1% California SDI rate instead of the applicable 1.3% rate. Its $579 state estimate should be $684.26, while its whole-dollar component rounding further obscures the cent-accurate total." -us,scenario_022,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a 1.1% California SDI rate instead of the applicable 1.3% rate. Its $579 state estimate should be $684.26, producing a traced total of $4,710.83." -us,scenario_022,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied a 0.9% California SDI rate instead of the 2026 rate of 1.3%. This understated the state contribution by $210.54, from $684.26 to $473.72." -us,scenario_022,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,The model included California SDI but used a 1.2% rate instead of 1.3%. Its state component is therefore $631.62 rather than $684.26. -us,scenario_022,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model omitted California's mandatory employee SDI contribution by asserting that California has no employee payroll tax. Federal Social Security and Medicare total $4,026.58, and the missing $684.26 state component brings the result to $4,710.83 under component-level rounding." -us,scenario_022,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model omitted California SDI and also arithmetically failed to sum the federal taxes it named: 6.2% plus 1.45% of $52,635 is $4,026.58, not $3,308.00. Adding the required $684.26 California contribution yields $4,710.83 under the traced component rounding." -us,scenario_022,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,The model applied an estimated 1.2% California SDI rate instead of the 2026 rate of 1.3%. This understated the state payroll tax by $52.64. -us,scenario_022,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,The model computed only the 7.65% federal FICA tax and omitted California's mandatory employee SDI contribution. The omitted state component is $684.26. -us,scenario_022,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,The model included only Social Security and Medicare taxes and omitted the mandatory California SDI contribution. The missing state payroll tax is $684.26. -us,scenario_022,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,The model included California SDI but applied a 1.2% rate instead of 1.3%. It therefore estimated about $632 rather than the traced $684.26 state contribution. -us,scenario_022,payroll_tax,grok-build-0.1,llm_error,thresholds_rates,False,The model used a projected 1.2% California SDI rate instead of the applicable 2026 rate of 1.3%. Its state component should be $684.26 rather than approximately $632. -us,scenario_022,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so the required output was missing." -us,scenario_022,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,The model applied a 1.2% California SDI rate instead of the 2026 rate of 1.3%. This produced $631.62 rather than $684.26 for the mandatory state contribution. -us,scenario_022,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,The model stopped after federal Social Security and Medicare taxes and omitted California's mandatory employee SDI contribution. The missing state component is $684.26. -us,scenario_022,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model included California SDI but used an approximate 1.1% rate and an obsolete wage-cap description instead of applying the 2026 1.3% rate to all $52,635. The state component is $684.26 rather than $578.99." +us,scenario_022,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model used a 1.2% California SDI rate instead of the 2026 rate of 1.3%, and its submitted $4,557.61 also contradicts its own stated $4,658.20 sum. California SDI is $684.26, producing total payroll tax of $4,710.83." +us,scenario_022,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly excluded mandatory California SDI, which contributes 1.3% of $52,635, or $684.26. It also misstated Medicare tax as $762.71 and submitted a value inconsistent with its own $4,026.08 calculation." +us,scenario_022,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model included California SDI but applied a 1.2% rate instead of the 2026 rate of 1.3%. The correct state contribution is $684.26 rather than $631.62, raising total payroll tax to $4,710.83." +us,scenario_022,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model explicitly excluded California SDI even though the requested output includes mandatory employee state payroll taxes. Adding the $684.26 SDI contribution to its federal FICA calculation yields $4,710.83." +us,scenario_022,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model omitted California’s mandatory employee SDI contribution of $684.26. It also rounded the federal subtotal of $4,026.58 down to $4,026.00 without justification; the complete total is $4,710.83." +us,scenario_022,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied a stale 1.1% California SDI rate instead of the 2026 rate of 1.3%. SDI is $684.26 rather than $578.99, so total payroll tax is $4,710.83." +us,scenario_022,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model wrongly treated California SDI as nonmandatory and omitted its $684.26 employee contribution. Its submitted $3,264.37 also contradicts its own federal FICA subtotal of $4,026.58; the complete payroll tax is $4,710.83." +us,scenario_022,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model’s $579 California SDI estimate applies a 1.1% rate rather than the 2026 rate of 1.3%. The correct SDI amount is $684.26, and the unrounded component sum is $4,710.83." +us,scenario_022,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a 1.1% California SDI rate instead of the 2026 rate of 1.3%. Replacing $578.99 with $684.26 produces total payroll tax of $4,710.83." +us,scenario_022,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model calculated only federal Social Security and Medicare taxes and omitted California’s mandatory employee SDI contribution. The missing state component is $684.26, making the total $4,710.83." +us,scenario_022,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a 1.1% California SDI rate instead of the applicable 1.3% rate. California SDI is $684.26 rather than about $579, yielding $4,710.83 in total payroll tax." +us,scenario_022,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a 1.1% California SDI rate instead of the applicable 1.3% rate. The state contribution is $684.26, so the three payroll-tax components total $4,710.83." +us,scenario_022,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied a 0.9% California SDI rate instead of the 2026 rate of 1.3%. SDI equals $684.26 rather than $473.72, and total payroll tax equals $4,710.83." +us,scenario_022,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"The model included California SDI but used 1.2% rather than the 2026 rate of 1.3%. The correct SDI contribution is $684.26, producing a $4,710.83 total." +us,scenario_022,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model incorrectly asserted that California imposes no employee state payroll tax and therefore omitted mandatory SDI. Adding the $684.26 California contribution to $4,026.58 of federal FICA yields $4,710.83." +us,scenario_022,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model omitted California SDI and also miscomputed the stated 7.65% federal FICA calculation: Social Security plus Medicare is $4,026.58, not $3,308.00. Adding $684.26 of SDI gives $4,710.83." +us,scenario_022,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"The model estimated California SDI at 1.2% instead of applying the 2026 rate of 1.3%. The correct SDI amount is $684.26, bringing total payroll tax to $4,710.83." +us,scenario_022,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model stopped after federal Social Security and Medicare taxes and omitted California’s mandatory employee SDI contribution. Federal FICA is $4,026.58 and SDI is $684.26, yielding $4,710.83." +us,scenario_022,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model included only Social Security and Medicare and omitted California’s mandatory employee SDI tax. The omitted 1.3% contribution is $684.26, producing total payroll tax of $4,710.83." +us,scenario_022,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"The model included California SDI but used a 1.2% rate instead of 1.3%. SDI is $684.26 rather than about $632, so total payroll tax is $4,710.83." +us,scenario_022,payroll_tax,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a projected 1.2% California SDI rate instead of the applicable 2026 rate of 1.3%. The correct state contribution is $684.26 and the total is $4,710.83." +us,scenario_022,payroll_tax,inkling,llm_error,thresholds_rates,False,"The model estimated California SDI at about 1.2% rather than applying the 2026 rate of 1.3%. Using $684.26 of SDI with the two federal components yields $4,710.83." +us,scenario_022,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required output contract." +us,scenario_022,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,"The model applied a 1.2% California SDI rate instead of the 2026 rate of 1.3%. The correct SDI contribution is $684.26 rather than $631.62, making total payroll tax $4,710.83." +us,scenario_022,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model calculated federal Social Security and Medicare taxes but omitted California’s mandatory employee SDI contribution. Adding $684.26 of SDI to the $4,026.58 federal subtotal yields $4,710.83." +us,scenario_022,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model included California SDI but applied an obsolete 1.1% rate and referenced an inapplicable capped wage base. At 1.3% of $52,635, SDI is $684.26 and total payroll tax is $4,710.83." +us,scenario_022,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model incorrectly stated that no mandatory California employee payroll tax applies and omitted SDI. The missing state contribution is $684.26, which raises the federal subtotal of $4,026.58 to $4,710.83." us,scenario_022,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-surviving-spouse joint brackets and a joint-equivalent standard deduction even though no qualifying child supports that filing status. It also deducted unreimbursed employee expenses and the full stated charitable gifts instead of using PolicyEngine's $23,676.67 allowed California itemized deduction." -us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented $134,529 of federal taxable income and a roughly $5,320 California income-tax credit from the Senior Property Tax Postponement program. Property-tax postponement is not a nonrefundable income-tax credit; the trace instead produces $67,847.75 of California taxable income and only $312.93 of exemption credits." -us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model treated the taxpayer as a qualifying surviving spouse and used the married-joint California rate schedule. The surviving-spouse fact alone does not establish that status without a qualifying child, so the single schedule applies to $67,847.75 of taxable income." -us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model used qualifying-surviving-spouse joint brackets and fabricated mortgage interest from the mortgage balance despite the instruction that unlisted amounts are zero. PolicyEngine uses single brackets and $23,676.67 of allowed itemized deductions, with no inferred mortgage-interest payment." -us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model correctly selected the single schedule but deducted the full $24,222 of stated property tax and charitable gifts. PolicyEngine limits the allowed charitable component to $14,847.67, giving total itemized deductions of $23,676.67 and taxable income of $67,847.75 rather than $67,302." -us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model treated the taxpayer as qualifying surviving spouse and therefore used married-joint brackets and a married personal exemption credit. With no qualifying child, the single schedule applies; the applicable nonrefundable aged and personal exemption credits total $312.93, not $588." -us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-widow joint brackets solely from the surviving-spouse flag and described the senior credit as embedded in the brackets. PolicyEngine applies the single schedule and separately subtracts $312.93 of aged and personal exemption credits from $2,752.58." -us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted $6,168 of unreimbursed employee expenses, reducing taxable income to $61,134. PolicyEngine's allowed California itemized deductions total $23,676.67 and produce taxable income of $67,847.75." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included employee business expenses in a $30,742.52 itemized deduction. The allowed California itemized deduction is $23,676.67, so taxable income is $67,847.75 and schedule tax is $2,752.58 before $312.93 of credits." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $1,175 estimate does not follow the California single-filer schedule on the traced taxable income. Applying that schedule to $67,847.75 produces $2,752.58 before $312.93 of nonrefundable exemption credits." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model's $30,743 itemized deduction includes expenses that are absent from PolicyEngine's allowed deduction. The correct allowed amount is $23,676.67, yielding $67,847.75 of taxable income rather than $60,781." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used $30,743 of itemized deductions and therefore understated taxable income at $60,781. PolicyEngine allows $23,676.67, producing $67,847.75 of taxable income, $2,752.58 of schedule tax, and $312.93 of credits." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model expressly included employee business expenses in its $30,743 deduction. PolicyEngine's California itemized deduction is $23,676.67, so those extra expenses cannot reduce taxable income to $60,781." -us,scenario_022,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model replaced the listed $4,550 rental income with $24,556 and invented rental depreciation, mortgage interest, and a $1,000 taxable Social Security amount. The traced federal AGI is $109,149.16 and California AGI after the Social Security subtraction is $91,524.41." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model incorrectly treated age and deductions as eliminating California tax. The traced $67,847.75 of taxable income generates $2,752.58 under the California schedule, and the $312.93 exemption credits do not fully offset it." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The submitted $1,036 implies an incorrect rate schedule or excessive credits. The California single schedule produces $2,752.58 on $67,847.75, followed by only $312.93 of nonrefundable credits." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model used joint or qualifying-surviving-spouse brackets and $60,781 of taxable income. With no qualifying child, PolicyEngine uses single brackets, and allowed deductions leave $67,847.75 taxable." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's terse estimate does not apply the traced schedule and credits correctly. Tax on $67,847.75 is $2,752.58 before $312.93 of nonrefundable aged and personal exemption credits, not $2,893 after credits." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-surviving-spouse rates and included employee expenses among itemized deductions. PolicyEngine instead uses the single schedule and $23,676.67 of allowed itemized deductions." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model's surviving-spouse tax calculation used the wrong filing-status pathway. The surviving-spouse flag without a qualifying child does not trigger joint-equivalent rates; single-filer tax before credits is $2,752.58." -us,scenario_022,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model defaulted to zero instead of applying the California computation. The traced taxable income of $67,847.75 produces positive schedule tax of $2,752.58, which remains $2,439.65 after nonrefundable credits." -us,scenario_022,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $30,743 itemized deduction, including an unsupported employee-expense deduction, and subtracted only the personal exemption credit. PolicyEngine allows $23,676.67 and subtracts $312.93 in combined aged and personal exemption credits." -us,scenario_022,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model deducted $6,521 of unreimbursed employee expenses after a 2% floor, lowering taxable income to $60,781. PolicyEngine's allowed itemized deductions are $23,676.67, and it also applies $312.93 of nonrefundable exemption credits rather than none." -us,scenario_022,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract." -us,scenario_022,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model deducted the full $15,393 of stated charitable gifts, producing $24,222 of itemized deductions and $67,302 of taxable income. PolicyEngine's allowed charitable deduction is $14,847.67, total itemized deductions are $23,676.67, and combined exemption credits are $312.93 rather than roughly $144." -us,scenario_022,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model incorrectly concluded that deductions reduce California taxable income to zero. The trace leaves $67,847.75 taxable, generating $2,752.58 before $312.93 of nonrefundable credits." -us,scenario_022,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model misstated California's progressive brackets as beginning at 4%, then arithmetically combined its bracket amounts and credits into a result that does not follow from its own figures. It also used an estimated standard deduction instead of the larger $23,676.67 allowed itemized deduction; the traced schedule tax is $2,752.58 and credits are $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly used joint qualifying-surviving-spouse brackets even though no dependent child establishes that filing status. It also deducted unreimbursed employee expenses and produced a final $3,162 figure that contradicts its own approximately $980 calculation." +us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented $134,529 of income and a $5,320 California income-tax credit from the Senior Property Tax Postponement program. Property-tax postponement is not a nonrefundable income-tax credit; the trace yields $67,847.75 of taxable income and only $312.93 of applicable exemption credits." +us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model applied married-joint qualifying-surviving-spouse brackets and credits without the required dependent child. The single schedule on $67,847.75 produces $2,752.58 before the $312.93 personal and senior exemption credits." +us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model treated the taxpayer as qualifying surviving spouse and applied joint brackets. It also invented mortgage interest from the outstanding mortgage balance, despite the instruction that unlisted expenses are zero." +us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used the correct single-rate framework but deducted the full $24,222 of property taxes and stated charitable gifts. The allowable California itemized deduction is $23,676.67, leaving $67,847.75 taxable, and the applicable nonrefundable exemption credits total $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model incorrectly assigned qualifying-surviving-spouse status and used married-joint brackets and a joint personal exemption credit. With no qualifying child, the single schedule applies, and the personal-plus-senior credits total $312.93 rather than $588." +us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-surviving-spouse brackets solely from the surviving-spouse fact and even described the senior exemption credit as embedded in the brackets. The taxpayer instead uses single brackets, followed by separate personal and senior exemption credits totaling $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted $6,168 of unreimbursed employee expenses, producing $61,134 of taxable income instead of $67,847.75. The traced California itemized deduction is $23,676.67 and does not include that employee-expense deduction." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model deducted $30,742.52 by including an employee-business-expense deduction that the trace does not allow. Allowable itemized deductions are $23,676.67, so taxable income is $67,847.75 rather than $60,781.48." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies an understated California rate-schedule calculation on the single filer's taxable income. The traced calculation applies the single schedule to $67,847.75 for $2,752.58, then subtracts $312.93 of exemption credits." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used $30,743 of itemized deductions, including the disallowed employee-business-expense amount, and therefore understated taxable income. The allowable deduction is $23,676.67 and taxable income is $67,847.75." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced California AGI by $30,743 instead of the traced $23,676.67 allowable itemized deduction. That error lowered taxable income from $67,847.75 to $60,781 and understated scheduled tax." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model expressly included unreimbursed employee business expenses in a $30,743 itemized deduction. The trace allows only $23,676.67, comprising $14,847.67 of charitable contributions and $8,829 of real-estate taxes." +us,scenario_022,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model replaced the listed $4,550 rental income with $24,556 and invented mortgage-interest and Social Security amounts, corrupting AGI at the outset. California AGI is $91,524.41 and taxable income after the traced itemized deduction is $67,847.75." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model incorrectly treated age and deductions as eliminating California tax. The deductions leave $67,847.75 taxable, generating $2,752.58 of scheduled tax, and the $312.93 exemption credits do not reduce it to zero." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The submitted $1,036 implies that the model understated the rate-schedule tax or overstated nonrefundable credits. The correct steps yield $2,752.58 before credits and only $312.93 of personal and senior exemption credits." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model used joint qualifying-surviving-spouse rates without a qualifying dependent child and started from taxable income reduced by an impermissible employee-expense deduction. The correct single-filer taxable income is $67,847.75, with $312.93 rather than about $486 of exemption credits." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's $2,893 exceeds the traced $2,752.58 scheduled tax even before the applicable credits, so it misapplied the brackets or failed to subtract all exemption credits. Personal and senior credits reduce scheduled tax by $312.93 to $2,439.65." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model incorrectly applied qualifying-surviving-spouse rates and included employee expenses among itemized deductions. With no qualifying child, single brackets apply to $67,847.75, and allowable nonrefundable credits total $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,The model used a surviving-spouse tax computation despite the absence of the dependent child required for qualifying-surviving-spouse filing status. It also omitted the separate senior exemption credit from its stated credit calculation. +us,scenario_022,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model defaulted to zero instead of applying the California tax schedule. Taxable income remains $67,847.75 after deductions, and $312.93 of credits leaves $2,439.65 due." +us,scenario_022,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model included unreimbursed employee expenses in a $30,743 itemized deduction, reducing taxable income to $60,781. The traced allowable deduction is $23,676.67, and it also omitted the senior portion of the $312.93 exemption credits." +us,scenario_022,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model deducted $6,521 of unreimbursed employee expenses, which are absent from the traced allowable itemized deductions. It then failed to subtract the applicable personal and senior exemption credits totaling $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model understated California AGI and included an employee-expense deduction, producing only about $59,862 of taxable income. The trace yields $91,524.41 of California AGI, $23,676.67 of itemized deductions, and $67,847.75 taxable." +us,scenario_022,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no output or explanation for the requested variable. +us,scenario_022,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model deducted the full $24,222 sum of stated charity and property taxes instead of the traced allowable $23,676.67. It also subtracted only about $144 of credits, omitting the full personal-and-senior credit total of $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model incorrectly concluded that deductions eliminate California taxable income. They reduce California AGI of $91,524.41 only to $67,847.75, on which the scheduled tax remains $2,752.58 before exemption credits." +us,scenario_022,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a fabricated rate schedule beginning at 4% and arithmetically produced components totaling over $6,000 before asserting $1,543.86. California's actual single schedule produces $2,752.58 on $67,847.75, and applicable credits are $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly treated deductions and the senior exemption credit as fully offsetting the liability. Taxable income remains $67,847.75, and the $312.93 personal and senior credits reduce $2,752.58 only to $2,439.65." us,scenario_022,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model invented an American Opportunity Tax Credit despite the required qualified education expenses being zero, and it awarded a Saver’s Credit without reducing eligible contributions by the $8,000 retirement distribution. Neither credit offsets the $643.43 tentative tax." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $8,000 taxable 403(b) distribution from gross income and used an incorrect $14,600 standard deduction. Including the distribution and applying the $16,100 deduction leaves $6,434.34 of taxable income rather than zero." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model treated enrollment and Form 1098-T facts as sufficient to generate the maximum American Opportunity Tax Credit. Qualified education expenses are zero under the prompt, so no AOTC offsets the $643.43 tax." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model awarded a nonrefundable American Opportunity Tax Credit solely from the student eligibility facts. With no listed qualified tuition or education expenses, the credit base is zero and the $643.43 tentative tax remains." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model applied up to $1,500 of nonrefundable American Opportunity Tax Credit without any qualified education expenses. The listed enrollment documentation establishes student status but does not create a credit amount, so tax remains $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly set the AOTC to zero but estimated the 2026 single standard deduction as $15,350. The applicable deduction is $16,100, which reduces taxable income to $6,434.34 and tax to $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model incorrectly used an American Opportunity Tax Credit to eliminate the residual liability even though no qualified education expenses were listed. It also misstated $17,443 plus $8,000 as $26,443; the traced income and deductions yield $6,434.34 of taxable income and $643.43 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $8,089 employer-sponsored insurance premium from income and then used an $8,350 standard deduction. The trace instead produces AGI of $22,534.34 and applies the $16,100 standard deduction, yielding $643.43 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,other,False,"The model awarded a 50% Saver’s Credit without applying the rule that recent retirement distributions reduce eligible contributions. The $8,000 taxable 403(b) distribution exceeds the listed contributions, leaving no Saver’s Credit to offset the $643.43 tax." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model’s assertion that taxable income is zero omits income or overstates deductions. AGI of $22,534.34 minus the $16,100 standard deduction leaves $6,434.34 taxable, producing $643.43 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"The model incorrectly used the standard deduction and Saver’s Credit to eliminate all liability. The standard deduction leaves $6,434.34 taxable, and the $8,000 retirement distribution reduces Saver’s Credit-eligible contributions to zero." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $14,446 by subtracting the employer-sponsored insurance premium from income, then incorrectly applied a Saver’s Credit. AGI is $22,534.34, and the $8,000 retirement distribution eliminates the Saver’s Credit contribution base." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model subtracted a $200 Saver’s Credit without reducing eligible retirement contributions by the $8,000 taxable 403(b) distribution. That distribution exceeds the contributions, so the Saver’s Credit is zero and the full $643.43 remains." -us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly rejected both the Saver’s Credit and AOTC but estimated the standard deduction as $15,400. Applying the $16,100 deduction to AGI of $22,534.34 gives $6,434.34 of taxable income and $643.43 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model used AOTC-related enrollment facts to eliminate the remaining tax without any listed qualified education expenses. The AOTC amount is zero, leaving $643.43 after the standard deduction and rate schedule." -us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model applied a partial American Opportunity Tax Credit to reduce the liability to $121 despite zero listed qualified education expenses. No AOTC is generated, so the tax is $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated wages below the standard deduction as decisive and failed to incorporate the taxable $8,000 403(b) distribution into the final taxable-income calculation. Total traced AGI exceeds the $16,100 deduction by $6,434.34, producing $643.43 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a post-TCJA reversion with an approximately $8,500 standard deduction and a personal exemption. The applicable 2026 computation instead uses the $16,100 single standard deduction and no personal exemption, leaving $6,434.34 taxable and $643.43 due." -us,scenario_023,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly reverted to an approximately $8,500 standard deduction and a 10%/15% rate schedule. The applicable 2026 single deduction is $16,100, and all $6,434.34 of taxable income falls in the 10% bracket." -us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required output contract." -us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required output contract." -us,scenario_023,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model awarded the nonrefundable portion of the American Opportunity Tax Credit despite zero listed qualified education expenses. The student-status facts alone do not establish a credit base, so no AOTC offsets the $643.43 tax." -us,scenario_023,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model used a nonrefundable American Opportunity Tax Credit to eliminate tax even though the prompt lists no qualified education expenses. It also estimated the standard deduction instead of applying $16,100; the correct taxable income is $6,434.34 and no education credit reduces the resulting $643.43." -us,scenario_023,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model correctly identified zero qualified education expenses and zero EITC, then overrode those facts by inventing “typical qualified expenses” and awarding the maximum $1,000 refundable AOTC. Unlisted tuition is zero, so the refundable AOTC expense base is zero." -us,scenario_023,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated AOTC enrollment and documentation conditions as sufficient without any qualified education expenses, even though unlisted tuition is zero. It also called the entire $2,500 AOTC refundable, while only 40%, capped at $1,000, can be refundable even with sufficient expenses." -us,scenario_023,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,The model invented an approximately $400 refundable AOTC despite the zero qualified-expense base and added a residual EITC after acknowledging that AGI phases the childless credit to zero. Both components are zero. -us,scenario_023,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model assumed maximum qualified education expenses and awarded the $1,000 refundable AOTC. The prompt sets unlisted tuition and other qualified education expenses to zero, so AOTC is zero." -us,scenario_023,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model awarded the maximum refundable AOTC without qualified education expenses and retained a residual childless EITC despite AGI exceeding the phaseout endpoint. Its stated $1,000 plus approximately $400 also does not arithmetically support the submitted $2,400." -us,scenario_023,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model calculated EITC solely from a reduced earned-income figure and failed to apply the separate AGI limitation. The taxable 403(b) distribution pushes AGI beyond the childless EITC phaseout endpoint, reducing the credit to zero." -us,scenario_023,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model phased the childless EITC using wages alone and ignored the higher AGI that includes the taxable $8,000 distribution. EITC uses the limiting result based on earned income or AGI, and the AGI calculation phases the credit fully to zero." -us,scenario_023,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model awarded AOTC from eligibility flags without any qualified education expenses, which are zero under the prompt’s default rule. It also incorrectly treated the full $2,500 maximum AOTC as refundable instead of applying the 40% refundable share capped at $1,000." -us,scenario_023,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model declared the head eligible for childless EITC based on earned income while omitting the AGI test. Including the taxable 403(b) distribution places AGI beyond the 2026 phaseout endpoint, yielding zero EITC." -us,scenario_023,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,The model assigned an approximately $503 childless EITC without applying the AGI-based phaseout. The taxable distribution raises AGI enough to eliminate the credit. -us,scenario_023,federal_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated AOTC enrollment, credential, and documentation flags as establishing the maximum credit. Qualified education expenses are zero, so the refundable AOTC is zero regardless of those eligibility flags or income." -us,scenario_023,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used approximately $19,000 as the childless EITC phaseout starting point, rather than the much lower single-filer phaseout threshold, and therefore retained $380. Applying the 2026 schedule to AGI of approximately $22,535 exhausts the credit." -us,scenario_023,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, violating the required output contract." -us,scenario_023,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, violating the required output contract." -us,scenario_023,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model assumed enough qualified expenses for a $1,000 refundable AOTC even though the expense base is zero. It also used an erroneous approximately $19,104 childless EITC phaseout start, retaining $387 when the AGI-based credit is fully phased out." -us,scenario_023,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model invented a $1,000 refundable AOTC despite zero qualified education expenses and calculated EITC from wages without applying the higher AGI from the taxable distribution. Its reasoning totals $1,112, but it submitted $3,925, so the submitted value also contradicts its own final computation." -us,scenario_023,free_school_meals_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model excluded the $8,000 taxable 403(b) distribution and compared wages alone with the federal income cutoff. Countable income is $25,442.65, or 1.59 times the federal poverty guideline, above the federal free-meal income threshold; moreover, FREE-tier qualification does not produce positive support without a school-aged student." -us,scenario_023,free_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly identified that the household has no child or school-aged student, then discarded that requirement and treated household-level FREE-tier qualification as positive meal support. California universal eligibility and SNAP categorical eligibility establish the FREE tier, but no eligible student exists to generate an annual benefit." -us,scenario_023,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model submitted no answer or explanation for the requested variable, violating the required output contract." -us,scenario_023,head_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly treated California's CHIP income ceiling as extending coverage to a disabled 28-year-old adult and conflated CHIP with broader Medi-Cal pathways. The head is already Medicaid-eligible through the WORKING_DISABLED_BUY_IN category, so the categorical Medicaid exclusion makes CHIP eligibility false regardless of the cited 266% FPL threshold or employer-sponsored insurance." -us,scenario_023,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for head_chip_eligible, violating the required output contract. The required result is 0 because the head is Medicaid-eligible through WORKING_DISABLED_BUY_IN and therefore categorically excluded from CHIP." -us,scenario_023,head_medicaid_eligible,claude-opus-5,llm_error,categorical_eligibility,False,The model treated California's 138% FPL adult expansion limit as the only Medicaid pathway. It omitted the working-disabled buy-in category established by the head's disability and employment. -us,scenario_023,head_medicaid_eligible,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model denied eligibility solely because MAGI exceeded 138% FPL. It failed to evaluate the separate working-disabled buy-in pathway, for which the MAGI adult expansion cutoff is not controlling." -us,scenario_023,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model evaluated only the adult expansion and SSI-linked pathways, then concluded that excess income defeated both. It omitted the working-disabled buy-in category, which independently makes this employed disabled head eligible without SSI receipt." -us,scenario_023,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The unexplained denial omitted the working-disabled buy-in pathway. The head is disabled and employed, producing the WORKING_DISABLED_BUY_IN category and Medicaid eligibility." -us,scenario_023,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model disregarded the explicitly listed disability and work facts as inadequate Medicaid-category details. Those facts establish the working-disabled buy-in pathway, so no additional disability fact or SSI receipt is required for the engine's eligibility determination." -us,scenario_023,head_medicaid_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model applied the income limit for ordinary adult Medi-Cal and stopped after finding income above it. It failed to apply the distinct working-disabled buy-in category available to this disabled worker. -us,scenario_023,head_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model stated that no separate Medicaid category was established despite the listed disability, wages, hourly work, and weekly hours. Those facts establish the working-disabled buy-in category, independently of the 138% FPL expansion test." -us,scenario_023,head_medicaid_eligible,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model limited its analysis to expansion Medicaid and SSI-linked eligibility. It omitted the working-disabled buy-in pathway, which covers the employed disabled head even though SSI is zero and MAGI is 1.41 times FPL." -us,scenario_023,head_medicaid_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model treated income above the adult expansion limit as dispositive. It never applied the working-disabled buy-in category triggered by the head's disability and employment. -us,scenario_023,head_medicaid_eligible,grok-4.5,llm_error,categorical_eligibility,False,The model checked expansion Medicaid and a conventional aged-blind-disabled income pathway but omitted the working-disabled buy-in pathway. Failure under SSI-related income rules does not defeat eligibility under the separate buy-in category. -us,scenario_023,head_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model tested the 138% FPL expansion limit and a generic non-MAGI aged-blind-disabled limit, but not the working-disabled buy-in rules. The latter category independently establishes eligibility for this disabled worker." -us,scenario_023,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for head_medicaid_eligible. It therefore failed the required output contract rather than presenting a substantive eligibility calculation. -us,scenario_023,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for head_medicaid_eligible. It therefore failed the required output contract rather than presenting a substantive eligibility calculation. -us,scenario_023,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model assumed that exceeding 138% FPL and receiving no SSI exhausted the possible Medicaid pathways. It omitted the working-disabled buy-in category, which does not require SSI receipt and applies to this employed disabled head." -us,scenario_023,head_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model incorrectly treated the generic disability flag as a proxy for SSDI-based Medicare entitlement. At age 28, disability alone does not satisfy the under-65 Medicare pathway without qualifying disability-benefit entitlement and the applicable waiting period or another medical criterion such as end-stage renal disease." -us,scenario_023,head_medicare_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model directly converted disability status into Medicare eligibility. The household facts establish neither qualifying disability-benefit entitlement nor another under-65 Medicare criterion, so the age-65 rule remains unsatisfied." -us,scenario_023,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required outputs contract." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model invented an American Opportunity Credit despite zero listed qualified education expenses and allowed a Saver’s Credit without reducing eligible contributions by the $8,000 retirement distribution. Neither credit offsets the $643.43 tax." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $8,000 taxable 403(b) distribution from gross income and used an incorrect $14,600 standard deduction. Including the distribution and applying the $16,100 deduction produces $6,434.34 of taxable income and $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied the nonrefundable American Opportunity Credit solely from the student-status facts. Qualified education expenses are zero under the prompt’s default rule, so the AOTC is zero and does not offset the $643.43 tax." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated enrollment, credential, and 1098-T facts as establishing a maximum American Opportunity Credit without any qualified education expenses. With expenses defaulted to zero, the AOTC is zero and the $643.43 liability remains." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model applied up to $1,500 of nonrefundable American Opportunity Credit without any listed qualified education expenses. Zero expenses produce zero AOTC, leaving $643.43 after the standard deduction and rate schedule." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model estimated the 2026 single standard deduction as $15,350 instead of applying the $16,100 statutory amount. The correct deduction reduces taxable income to $6,434.34 and tax to $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that the American Opportunity Credit eliminates the residual tax even though qualified education expenses default to zero. It also misstated $17,443 plus $8,000 as $26,443; the correct AGI and deduction yield $643.43 with no AOTC offset." +us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the $8,089 employer-sponsored insurance premium from wages and used an $8,350 standard deduction. The trace reduces wages only by the traditional 401(k), then applies the $16,100 standard deduction, producing $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model allowed a $1,000 Saver’s Credit without subtracting the $8,000 taxable retirement distribution from eligible retirement contributions. That distribution exceeds the contributions and reduces the creditable contribution amount to zero, so the $643.43 tax is not offset." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies that the model omitted taxable income or over-deducted it. Correctly including the $8,000 taxable 403(b) distribution yields $22,534.34 of AGI and $6,434.34 after the $16,100 standard deduction, not zero." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model incorrectly used the Saver’s Credit to eliminate the tax. The $8,000 retirement distribution exceeds the eligible retirement contributions and reduces the creditable contribution amount to zero; the standard deduction alone leaves $6,434.34 taxable." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model’s $14,446 AGI deducts the $8,089 employer-sponsored insurance premium even though the traced wage reduction consists only of the traditional 401(k) contribution. It also applied a Saver’s Credit despite the $8,000 distribution eliminating eligible net contributions." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model subtracted a $200 Saver’s Credit without netting the $8,000 retirement distribution against retirement contributions. The net creditable contribution is zero, and applying the $16,100 standard deduction produces $643.43 rather than $553.50." +us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly denied both credits but used an estimated $15,400 standard deduction instead of $16,100. The additional $700 deduction lowers taxable income to $6,434.34 and tax to $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used an AOTC-related nonrefundable credit without any listed qualified education expenses. Those expenses default to zero, so the AOTC is zero and cannot eliminate the $643.43 tax." +us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model’s $121 result reflects an unsupported partial American Opportunity Credit against the ordinary tax. Qualified education expenses are zero, so no AOTC applies and the full $643.43 remains." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated wages below the standard deduction as dispositive and failed to carry the $8,000 taxable 403(b) distribution through taxable income. After the retirement deductions and $16,100 standard deduction, taxable income is $6,434.34 rather than zero." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA reversion with an approximately $8,500 standard deduction and a personal exemption. The applicable 2026 single standard deduction is $16,100, producing $6,434.34 of taxable income and $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly reverted to an $8,500 standard deduction and a 15% second bracket after assuming TCJA expiration. The 2026 computation uses a $16,100 standard deduction, leaving all $6,434.34 of taxable income in the 10% bracket." +us,scenario_023,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model wrongly treated the $8,089 employer-sponsored insurance premium as an additional pre-tax wage deduction. Excluding that unsupported deduction gives $22,534.34 of AGI and $6,434.34 of taxable income." +us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output. +us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output. +us,scenario_023,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model correctly recognized that education expenses were unlisted elsewhere in the prompt but nevertheless applied the maximum nonrefundable AOTC here. With qualified education expenses equal to zero, the AOTC is zero; it also used an understated standard deduction instead of $16,100." +us,scenario_023,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model applied the American Opportunity Credit based on enrollment and documentation facts without any qualified education expenses. Expenses default to zero, so the credit is zero; the correct $16,100 standard deduction and IRA deduction yield $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that income falls below the standard deduction, disregarding the $8,000 taxable 403(b) distribution. Correct AGI is $22,534.34, leaving $6,434.34 taxable after the $16,100 standard deduction." +us,scenario_023,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model correctly established that qualified education expenses are zero and that the refundable AOTC therefore equals zero, then discarded that result by assuming unspecified “typical” expenses. The prompt requires unlisted expenses to be zero, so the submitted $1,000 AOTC has no credit base." +us,scenario_023,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model treated the $2,500 total AOTC maximum as fully refundable, although only 40%—at most $1,000—is refundable. More fundamentally, the household has zero listed qualified education expenses, so neither portion of the AOTC arises, and it also omitted the taxable $8,000 distribution from AGI." +us,scenario_023,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model invented a refundable AOTC amount despite zero qualified education expenses and added an unspecified “minimal EITC.” The taxable 403(b) distribution places income beyond the childless EITC range, so both components are zero." +us,scenario_023,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,The model assumed maximum qualified education expenses even though the prompt explicitly sets every unlisted numeric input to zero. Enrollment and documentation flags establish status conditions but do not create an AOTC expense base. +us,scenario_023,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model assigned $1,000 of refundable AOTC without any qualified education expenses and retained a residual EITC after acknowledging that income phases the childless credit to zero. Its stated $1,000 plus approximately $400 also does not produce the submitted $2,400." +us,scenario_023,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced EITC earned income by employer health premiums and desired retirement contributions and then applied only the phase-in formula. It failed to apply the separate AGI limitation, including the taxable $8,000 403(b) distribution, which eliminates the childless EITC." +us,scenario_023,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model phased the childless EITC using wages alone. EITC must also be limited using AGI, and the taxable $8,000 403(b) distribution raises income beyond the applicable childless EITC ceiling." +us,scenario_023,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model confused the $2,500 total AOTC maximum with a fully refundable credit; the refundable share is capped at $1,000. Because no qualified education expenses are listed, the actual AOTC base is zero regardless of the eligibility flags." +us,scenario_023,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model placed the head in the childless EITC phaseout range without including the taxable $8,000 403(b) distribution in the controlling AGI test. Including that distribution moves income beyond the phaseout endpoint and reduces the EITC to zero." +us,scenario_023,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,The submitted $503 is consistent with applying the 7.65% childless EITC phase-in to a reduced wage figure while omitting the AGI phaseout test. The taxable 403(b) distribution causes the AGI test to eliminate the credit. +us,scenario_023,federal_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the AOTC enrollment and documentation conditions as sufficient for the maximum refundable amount. Qualified education expenses are also required, and the prompt sets that unlisted expense input to zero." +us,scenario_023,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used roughly $19,000 as the start of the childless EITC phaseout instead of the applicable much lower phaseout threshold. Applying the proper schedule to AGI that includes the taxable distribution phases the credit completely to zero." +us,scenario_023,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model correctly excluded AOTC for lack of qualified expenses but calculated EITC solely as 7.65% of its reduced earned-income figure. It omitted the AGI phaseout test, under which the taxable 403(b) distribution eliminates the childless credit." +us,scenario_023,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. +us,scenario_023,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. +us,scenario_023,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model assumed a $1,000 refundable AOTC despite zero qualified education expenses and used an erroneously high childless EITC phaseout threshold. With no AOTC expense base and AGI beyond the proper EITC endpoint, both calculated components are zero." +us,scenario_023,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model assumed maximum AOTC expenses despite the prompt's zero-default rule and computed EITC from wages without applying the controlling AGI test that includes the taxable distribution. It then submitted $3,925 even though its own final component calculation totaled $1,112, so the submitted number is unsupported by its reasoning." +us,scenario_023,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a child and attributed the result to the Additional Child Tax Credit. The household contains only the 28-year-old head, so there is no qualifying child and no refundable CTC." +us,scenario_023,free_school_meals_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model excluded the $8,000 taxable 403(b) distribution and compared wages alone with the federal income cutoff; the trace’s applicable income is $25,442.65, or 1.59 times the poverty guideline. It also equated income eligibility with positive annual support even though this adult-only household has no school-meal recipient." +us,scenario_023,free_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly identified the adult-only household and the absence of a school-aged child, then incorrectly asserted that household-level income qualification produces free-meal support anyway. California’s universal and categorical pathways establish the FREE tier, but without a student receiving meals PolicyEngine returns no positive annual support." +us,scenario_023,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for the requested variable, violating the required submission contract." +us,scenario_023,head_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly treated California CHIP as covering disabled adults up to an extended 266% FPL threshold. It failed to apply the categorical exclusion for a person already Medicaid-eligible through the WORKING_DISABLED_BUY_IN pathway, which makes this head ineligible for CHIP regardless of the CHIP income thresholds." +us,scenario_023,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable output for head_chip_eligible. The required result was 0 because Medicaid eligibility under WORKING_DISABLED_BUY_IN categorically excludes the head from CHIP. +us,scenario_023,head_medicaid_eligible,claude-opus-5,llm_error,categorical_eligibility,False,The model treated the 138% FPL Medi-Cal adult-expansion ceiling as the only applicable pathway. It omitted the WORKING_DISABLED_BUY_IN category established by the head's disability and employment. +us,scenario_023,head_medicaid_eligible,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model compared MAGI solely with the 138% FPL expansion threshold and stopped. The head instead qualifies under the separate WORKING_DISABLED_BUY_IN category. +us,scenario_023,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,The model tested adult expansion and SSI-linked Medicaid but omitted the working-disabled buy-in pathway. Failure of the expansion and SSI-linked tests does not negate eligibility under WORKING_DISABLED_BUY_IN. +us,scenario_023,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model's unsupported ineligibility conclusion omitted the WORKING_DISABLED_BUY_IN category. The head's disability and employment place the person in that separate Medicaid eligibility pathway. +us,scenario_023,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model disregarded the expressly listed disability and work facts as insufficient Medicaid-category details. Those facts establish the WORKING_DISABLED_BUY_IN category used for eligibility. +us,scenario_023,head_medicaid_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model applied the ordinary adult Medi-Cal income limit to the head's wages and 403(b) distribution as though it exhausted eligibility. It failed to evaluate the separate WORKING_DISABLED_BUY_IN pathway. +us,scenario_023,head_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model incorrectly stated that no separate Medicaid category was established. The listed disability and employment establish the WORKING_DISABLED_BUY_IN category. +us,scenario_023,head_medicaid_eligible,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model considered only adult expansion and SSI-linked eligibility. It omitted WORKING_DISABLED_BUY_IN, which is distinct from SSI-linked Medicaid and applies to this employed disabled head." +us,scenario_023,head_medicaid_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model denied eligibility solely because MAGI exceeded the adult-expansion limit. It never applied the separate WORKING_DISABLED_BUY_IN category. +us,scenario_023,head_medicaid_eligible,grok-4.5,llm_error,categorical_eligibility,False,The model tested the expansion and aged-blind-disabled income pathways but failed to test the working-disabled buy-in pathway. The head qualifies through WORKING_DISABLED_BUY_IN regardless of failing those two alternatives. +us,scenario_023,head_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,The model limited its analysis to expansion Medicaid and a generic non-MAGI aged-blind-disabled test. It omitted the distinct WORKING_DISABLED_BUY_IN category that makes the employed disabled head eligible. +us,scenario_023,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no head_medicaid_eligible value or explanation, violating the required output contract." +us,scenario_023,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no head_medicaid_eligible value or explanation, violating the required output contract." +us,scenario_023,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated expansion coverage and SSI receipt as the only routes to Medicaid. It omitted WORKING_DISABLED_BUY_IN, which does not require current SSI receipt." +us,scenario_023,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The model applied a general California Medicaid income limit without evaluating categorical eligibility. The head qualifies through the separate WORKING_DISABLED_BUY_IN pathway. +us,scenario_023,head_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the generic disability flag as a proxy for SSDI-based Medicare entitlement. Disability alone does not confer Medicare eligibility; the facts provide neither the required disability-benefit entitlement pathway nor end-stage renal disease, and age 28 does not satisfy age-based eligibility." +us,scenario_023,head_medicare_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model incorrectly made disability by itself sufficient for under-65 Medicare eligibility. The head has no listed qualifying disability-benefit entitlement or end-stage renal disease and is under 65, so the correct eligibility value is 0." +us,scenario_023,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable head_medicare_eligible output. The required derivation yields 0 because the 28-year-old head has no listed Medicare-qualifying entitlement or medical criterion. us,scenario_023,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_023,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model used a $197.11 California SDI amount instead of the $226.75 state payroll-tax component. The federal components were essentially correct, so the understated state contribution caused the error." -us,scenario_023,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that California has no mandatory employee state payroll tax and omitted the $226.75 SDI contribution. Its submitted $1,335.85 also does not equal its own stated federal total of $1,334.38." -us,scenario_023,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model calculated California SDI as $209.32 using a 1.2% rate. PolicyEngine's California employee payroll-tax component is $226.75, leaving its total $17.41 too low." -us,scenario_023,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used a 0.9% California SDI rate and stated a $156.99 contribution instead of $226.75. Its submitted $1,467.32 also contradicts its own component sum of $1,490.38." -us,scenario_023,payroll_tax,claude-opus-5,llm_error,thresholds_rates,False,"The model used $209.32 for California SDI rather than the $226.75 state contribution. Its submitted $1,546 also differs from both its stated approximate total and the exact sum of its components." -us,scenario_023,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly retained the full wage base for FICA but calculated California SDI as $191.87. PolicyEngine includes $226.75 of California employee payroll tax, so the state component was understated by $34.88." -us,scenario_023,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,The model falsely treated California as having no mandatory employee-side state payroll tax. It omitted the $226.75 California SDI contribution and returned federal FICA alone. -us,scenario_023,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted $8,089 of employer-sponsored insurance premiums from the stated gross wages even though those premiums are not an employee pre-tax payroll deduction specified by the prompt. PolicyEngine uses $17,442.65 as the payroll-tax wage base and also applies a $226.75 California contribution." -us,scenario_023,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used $191.87 for California SDI based on a 1.1% rate. The applicable PolicyEngine state payroll-tax component is $226.75, producing the higher total of $1,561.12." -us,scenario_023,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model included only Social Security and Medicare and omitted California's mandatory employee SDI contribution of $226.75. It also overstated Medicare slightly relative to PolicyEngine's $252.92 calculation. -us,scenario_023,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The answer implies that the model reduced payroll-tax wages by the $8,089 employer-sponsored insurance premium. PolicyEngine instead uses $17,442.65 of employment income as the base and calculates $1,561.12 across federal FICA and California SDI." -us,scenario_023,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly reduced FICA and SDI wages from $17,443 to $9,354 by subtracting employer-sponsored insurance premiums. Those premiums are not a listed employee pre-tax wage deduction, and PolicyEngine applies payroll taxes to $17,442.65." -us,scenario_023,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model calculated California SDI as $191.87 using 1.1% of wages. PolicyEngine's California employee payroll-tax component is $226.75, so its total was understated by $34.86." -us,scenario_023,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,The model correctly kept retirement contributions in the payroll-tax base but used $209.32 for California SDI. PolicyEngine calculates the state employee payroll-tax component as $226.75. -us,scenario_023,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly excluded all mandatory California employee payroll tax, omitting the $226.75 SDI contribution. Its submitted $1,375.966 also does not equal 7.65% of $17,443, which is about $1,334.39." -us,scenario_023,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model miscomputed 6.2% of $17,443 as about $1,070.10 instead of about $1,081.47 and then produced a federal total inconsistent with its stated arithmetic. It also omitted the $226.75 California SDI contribution." -us,scenario_023,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"The model used an estimated 1.2% California SDI amount of $209.32. PolicyEngine's state employee payroll-tax component is $226.75, making the submitted total $17.41 too low." -us,scenario_023,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,The model calculated only the 7.65% federal Social Security and Medicare taxes. It omitted California's mandatory $226.75 employee SDI contribution. -us,scenario_023,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model returned rounded federal FICA alone and omitted the $226.75 California SDI contribution. PolicyEngine includes that mandatory state employee payroll tax in the requested output. -us,scenario_023,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"The model used a projected 1.2% California SDI contribution of $209.32 rather than PolicyEngine's $226.75 state component. Rounding its incorrect component sum to $1,544 compounded the discrepancy." -us,scenario_023,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $2,778 traditional 401(k) contribution from FICA and SDI wages. Traditional 401(k) deferrals remain subject to employee Social Security and Medicare taxes, and PolicyEngine uses the $17,442.65 employment-income base with a $226.75 California contribution." -us,scenario_023,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax value or explanation. It therefore failed the required structured-output contract. -us,scenario_023,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no payroll_tax value or explanation. It therefore failed the required structured-output contract. -us,scenario_023,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model included only Social Security and Medicare, rounded their total, and omitted California's $226.75 mandatory employee SDI contribution. The requested payroll_tax output expressly includes mandatory state employee payroll taxes." -us,scenario_023,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,The model's reasoning calculated and acknowledged a California SDI component but its submitted value included only federal FICA. It therefore omitted the state contribution entirely from the numeric output despite recognizing that it belongs in payroll_tax. +us,scenario_023,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model applied a 1.13% California SDI rate instead of the 1% rate used for 2026. California SDI contributes $226.75 on the $17,442.65 state payroll-tax wage base, bringing total payroll tax to $1,561.12." +us,scenario_023,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly asserted that California has no mandatory employee state payroll tax and omitted California SDI of $226.75. Its submitted $1,335.85 also does not equal its stated federal components, which sum to about $1,334.39 on rounded wages." +us,scenario_023,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied a 1.2% California SDI rate instead of 1%. The correct California employee contribution is $226.75, and the three payroll-tax components total $1,561.12." +us,scenario_023,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied a 0.9% California SDI rate instead of 1%, then submitted $1,467.32 even though its own listed components total $1,490.38. Applying the correct state rate to the traced wage base produces $226.75 of SDI and $1,561.12 in total payroll tax." +us,scenario_023,payroll_tax,claude-opus-5,llm_error,thresholds_rates,False,"The model used a 1.2% California SDI rate instead of 1% and then submitted a value inconsistent with its own component calculation. California SDI is $226.75, yielding total payroll tax of $1,561.12." +us,scenario_023,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"After correctly rejecting an unsupported Section 125 wage reduction, the model applied a 1.1% California SDI rate instead of 1%. The state contribution is $226.75 on $17,442.65, not $191.87." +us,scenario_023,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model falsely treated California SDI as repealed and included only federal FICA. California imposes a mandatory 1% employee SDI contribution of $226.75, so total payroll tax is $1,561.12." +us,scenario_023,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the $8,089 employer-sponsored insurance premium from stated gross wages, even though the prompt does not identify an employee Section 125 salary reduction. It also used a 1.2% SDI rate instead of 1%; payroll taxes apply to the traced $17,442.65 wage base." +us,scenario_023,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a 1.1% California SDI estimate instead of the applicable 1% rate. The traced California SDI amount is $226.75, and total payroll tax is $1,561.12." +us,scenario_023,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model calculated only Social Security and Medicare and omitted the mandatory California SDI contribution of $226.75. It also overstated Medicare slightly through arithmetic error; the traced federal components are $1,081.44 and $252.92." +us,scenario_023,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly reduced gross wages by the $8,089 employer-sponsored insurance premium without any stated employee pre-tax payroll deduction. Federal payroll taxes and 1% California SDI apply to $17,442.65, producing $1,561.12." +us,scenario_023,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly treated the $8,089 employer-sponsored insurance premium as an employee pre-tax wage deduction and reduced the payroll-tax base to $9,354. It also used 1.1% rather than 1% for California SDI; the correct base is $17,442.65." +us,scenario_023,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied a 1.1% California SDI rate instead of 1%. The correct state contribution is $226.75 and the total is $1,561.12." +us,scenario_023,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"The model correctly kept retirement deferrals in the FICA base but applied a 1.2% California SDI rate instead of 1%. California SDI is $226.75 on the traced wage base, producing a $1,561.12 total." +us,scenario_023,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model incorrectly omitted mandatory California SDI of $226.75. Its submitted $1,375.966 also does not equal 7.65% of the stated $17,443 wage amount, so both the state component and its arithmetic are wrong." +us,scenario_023,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model omitted California SDI and miscalculated Social Security: 6.2% of $17,443 is about $1,081.47, not $1,070.10. The traced federal components plus $226.75 of SDI total $1,561.12." +us,scenario_023,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"The model used an estimated 1.2% California SDI rate instead of 1%. The correct state contribution is $226.75, and total employee payroll tax is $1,561.12." +us,scenario_023,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,The model stopped after calculating federal Social Security and Medicare at 7.65% and omitted California SDI. The missing mandatory employee state payroll tax is $226.75. +us,scenario_023,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model included only Social Security and Medicare and omitted California's mandatory employee SDI contribution. Adding the traced $226.75 state component to the federal components yields $1,561.12." +us,scenario_023,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"The model correctly retained retirement contributions in payroll wages but used a projected 1.2% California SDI rate instead of 1%. California SDI is $226.75, so the total is $1,561.12 rather than a rounded $1,544." +us,scenario_023,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $2,778 traditional 401(k) contribution from FICA-taxable wages; traditional 401(k) deferrals remain subject to Social Security and Medicare taxes. It also used a 1.1% SDI rate instead of 1%, whereas all three components use the traced $17,442.65 wage base." +us,scenario_023,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model improperly subtracted $8,089 of employer-sponsored insurance premiums from wages despite no stated employee Section 125 payroll deduction. It also applied 1.2% rather than 1% California SDI; the correct payroll-tax wage base is $17,442.65." +us,scenario_023,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, so it failed the required output contract." +us,scenario_023,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, so it failed the required output contract." +us,scenario_023,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model calculated only federal Social Security and Medicare and omitted California SDI. The missing 1% mandatory employee state contribution is $226.75, producing a total of $1,561.12." +us,scenario_023,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The reasoning calculated a California SDI component but the submitted value retained only federal FICA, omitting the state tax from the answer. It also used 1.1% rather than 1%; the correct California SDI amount is $226.75 and total payroll tax is $1,561.12." +us,scenario_023,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The submitted $261.65 is incompatible with Social Security and Medicare on $17,442.65 of wages, which alone equal $1,334.36 in the computation trace. The model failed to apply the stated wage base and also omitted $226.75 of California SDI." us,scenario_023,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_023,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated the generic disability flag as SNAP-recognized disability, deducted medical expenses, and allowed an uncapped $1,218 monthly excess-shelter deduction. The applicable capped shelter calculation leaves $876.51 of net income, not $345." -us,scenario_023,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It applied the ordinary 130% gross-income test as disqualifying and ignored categorical eligibility through TANF non-cash assistance. High rent is a deduction input, not a reason for SNAP ineligibility." -us,scenario_023,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,It treated `is_disabled` as qualification for the SNAP medical deduction and uncapped shelter deduction. Applying the applicable capped shelter calculation produces $876.51 in net income and only a small monthly allotment. -us,scenario_023,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It incorrectly granted both the disability medical deduction and an uncapped shelter deduction, then abandoned its own $1,788 calculation for an unsupported $2,549 figure. SNAP-recognized disability was not established, so the capped shelter computation controls." -us,scenario_023,snap,claude-opus-5,llm_error,categorical_eligibility,False,"It denied eligibility under a gross-income test and assumed the remaining deductions yielded zero. TANF non-cash categorical eligibility bypasses that shortcut, and the net-income calculation yields a positive allotment." -us,scenario_023,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It equated the generic disability flag with SNAP-recognized disability and therefore applied a medical deduction and uncapped $1,223.90 shelter deduction. The applicable shelter cap leaves $876.51 of monthly net income rather than $328.30." -us,scenario_023,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,It treated gross income above 130% of poverty as disqualifying and never applied TANF non-cash categorical eligibility. The household passes the resulting net-income test and receives a positive benefit. -us,scenario_023,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It deducted $747.42 per month of medical expenses even though the listed medical amounts are annual, then used an uncapped shelter deduction to force net income to zero. Those deductions do not apply on that basis, and net income is $876.51." -us,scenario_023,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,It incorrectly treated the household as entitled to disability medical and uncapped shelter deductions and added a utility allowance not listed in the facts. This forced net income to zero instead of the traced $876.51. -us,scenario_023,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its maximum-allotment answer implies it reduced net income to zero rather than applying the capped shelter computation. The correct monthly net income is $876.51, so the household receives only the maximum allotment minus a 30% contribution." -us,scenario_023,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It explicitly granted an uncapped shelter deduction and medical deduction solely from the generic disability flag. SNAP-recognized disability was not established, so those deductions cannot reduce countable net income to zero." -us,scenario_023,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,It estimated net income near $300 by applying uncapped shelter and medical deductions for a disabled household. The applicable capped shelter computation produces $876.51 of net income and a much smaller benefit. -us,scenario_023,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It based its estimate on uncapped shelter expenses for the generic disability flag. SNAP-recognized disability was not established, and the capped deduction leaves a substantially larger expected contribution." -us,scenario_023,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,It deducted annual medical expenses above $420 and the full uncapped excess shelter amount because it treated `is_disabled` as SNAP-recognized disability. The applicable capped shelter calculation leaves $876.51 per month of net income. -us,scenario_023,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,Its zero answer ignores TANF non-cash categorical eligibility and the traced net-income deductions. The household passes the net-income and asset tests and receives $461.34 annually. -us,scenario_023,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,It required separate listed evidence of SNAP receipt or eligibility despite the instruction to calculate eligibility and assume take-up. TANF non-cash categorical eligibility and the net-income test produce a positive benefit. -us,scenario_023,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It treated the generic disability flag as authorizing medical and uncapped excess-shelter deductions, reducing net income to zero. The applicable capped shelter calculation leaves $876.51 in monthly net income." -us,scenario_023,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It assumed disability made the high rent an uncapped shelter deduction and therefore awarded the maximum allotment. SNAP-recognized disability was not established, so the shelter cap applies and net income remains $876.51." -us,scenario_023,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It correctly identified the student work-hours pathway but incorrectly applied disability medical and uncapped shelter deductions. The capped shelter computation creates a 30% expected contribution of about $262.80 in most months, not a $202 monthly benefit." -us,scenario_023,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,It treated earnings and the retirement distribution as eliminating the benefit without applying TANF non-cash categorical eligibility and the full net-income calculation. The resulting net income passes the test and yields a positive allotment. -us,scenario_023,snap,grok-4.3,llm_error,categorical_eligibility,False,It asserted that income and assets exceed the limits even though bank assets are only $130 and categorical eligibility applies. The household passes the asset and net-income tests after the applicable deductions. -us,scenario_023,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"It classified the household as SNAP disabled, deducted medical expenses, and allowed the full $14,837 annual excess-shelter deduction. The generic disability flag does not establish that deduction status, and the applicable shelter cap leaves $876.51 monthly net income." -us,scenario_023,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It omitted the $8,000 taxable 403(b) distribution from income before shelter and then used an uncapped shelter deduction to reduce net income to zero. The distribution is countable unearned income and the applicable shelter deduction is capped." -us,scenario_023,snap,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It treated the generic disability flag as both a gross-test exemption and authorization for medical and uncapped shelter deductions, producing zero net income. The traced categorical-eligibility pathway and capped shelter calculation instead leave $876.51." -us,scenario_023,snap,kimi-k3,parse_contract_failure,missing_output,False,"It returned no SNAP value or explanation, so it failed the required output contract before any substantive calculation could be evaluated." -us,scenario_023,snap,minimax-m3,llm_error,categorical_eligibility,False,"It denied SNAP solely because its gross-income figure exceeded 130% of poverty and ignored TANF non-cash categorical eligibility. It also understated total gross income by not consistently including the full $8,000 taxable 403(b) distribution." -us,scenario_023,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It omitted the $8,000 taxable 403(b) distribution from gross income and then awarded the full maximum allotment despite stating net income of about $944, which would require subtracting a 30% contribution. Including countable unearned income and applying the traced deductions yields $876.51 net income and a small positive benefit." -us,scenario_023,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model treated the prompt's generic `is disabled` fact as establishing SSI disability and proceeded directly to earned- and unearned-income counting. PolicyEngine's SSI-specific categorical determination is false, so no federal SSI or California SSP benefit-rate calculation applies." -us,scenario_023,ssi,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no parseable SSI output or explanation, violating the required `submit_outputs` contract." -us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model rounded the tentative tax and exemption credit instead of applying the 2026 amounts. The exact $223.27 tax minus $156.47 of exemptions and the $60 renter credit leaves $6.80, so the credits do not fully offset liability." -us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model omitted the taxable $8,000 403(b) distribution and incorrectly treated CalEITC as a nonrefundable credit against this output. California AGI is $22,534.34, and the applicable nonrefundable exemption and renter credits reduce $223.27 to $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented an unspecified disability offset and treated the personal exemption credit as eliminating the tax. Only $156.47 of exemption credits and the $60 renter credit offset the $223.27 tentative tax, leaving $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model computed an approximate $233 tentative tax and then asserted that rounding eliminated the remaining liability. Using the $5,706 deduction and exact brackets gives $223.27, and subtracting $216.47 of credits leaves $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model's rounded deduction, bracket tax, and exemption credit erased a small positive balance. Exact computation yields $223.27 before credits and $216.47 of nonrefundable credits, leaving $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly awarded a second California exemption credit solely because the taxpayer is disabled. PolicyEngine applies $156.47 in total exemptions, not roughly $288; with the $60 renter credit, $6.80 remains." -us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invoked unspecified disability-related credits and then claimed the personal exemption exceeded tentative tax. The applicable credits are $156.47 of exemptions plus the $60 renter credit, which reduce $223.27 to $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model derived $14,446 of AGI by excluding the taxable $8,000 403(b) distribution and mishandling the listed retirement deductions. Correct AGI is $22,534.34 and taxable income is $16,828.34, producing $6.80 after credits." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used approximate deduction, bracket, and exemption values that overstated the residual tax. Exact tentative tax is $223.27 and exact nonrefundable credits are $216.47, leaving $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly stated that the standard deduction exceeds AGI. California AGI is $22,534.34, well above the $5,706 deduction, leaving $16,828.34 taxable and $6.80 after credits." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model incorrectly treated the personal exemption credit as fully offsetting the tax without computing the remaining balance. The $223.27 tentative tax exceeds the combined $216.47 exemption and renter credits by $6.80. -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated taxable income as $8,846, consistent with omitting taxable income or taking unsupported deductions. Taxable income is $16,828.34, and its $223.27 tax exceeds the $216.47 credits by $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model's $218 answer is essentially an estimate of tentative bracket tax and does not subtract the applicable nonrefundable credits. PolicyEngine subtracts $156.47 of exemptions and the $60 renter credit from $223.27, leaving $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model explicitly applied no nonrefundable state credits and also used stale deduction and bracket values. The taxpayer receives $156.47 of exemption credits and a $60 renter credit; after exact bracket tax, liability is $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,The model collapsed the calculation to zero without computing the narrow excess of tax over credits. The $223.27 tentative tax is $6.80 greater than the $216.47 of applicable nonrefundable credits. -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model incorrectly associated the federal American Opportunity Credit with California nonrefundable credits and produced an unsupported residual. The state calculation instead subtracts $156.47 of exemptions and the $60 renter credit from $223.27, yielding $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model replaced regular California income tax with a 2.5% early-distribution additional tax on the entire $8,000 distribution. This output follows the regular tax computation: $223.27 less $216.47 of nonrefundable credits equals $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,The model incorrectly concluded that the personal exemption and renter credit fully offset tentative tax. Their combined $216.47 is $6.80 less than the $223.27 tax. -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used projected and rounded deduction, bracket, and credit amounts, understating the residual by $2.80. Exact taxable income is $16,828.34, exact tentative tax is $223.27, and exact credits leave $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model treated the exemption and renter credits as fully eliminating the liability. The credits total $216.47 against $223.27 of tax, leaving $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly inferred zero tax from low income and the standard deduction. The deduction leaves $16,828.34 taxable, generating $223.27 before $216.47 of credits and $6.80 afterward." -us,scenario_023,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model's projected deduction, bracket cutoffs, and exemption credit left $9 because it did not use the exact 2026 parameters. The exact amounts are $223.27 of tax and $216.47 of credits, leaving $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,The model subtracted the $60 renter credit but omitted the $156.47 exemption credits. Applying both credits to the exact $223.27 tentative tax leaves $6.80. -us,scenario_023,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required output contract." -us,scenario_023,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required output contract." -us,scenario_023,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model added the traditional 401(k) and IRA contributions back to AGI even though they reduce the relevant income, inflating AGI from $22,534.34 to $25,443. It also omitted $216.47 of nonrefundable credits; the correct sequence leaves $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model failed to deduct the traditional retirement contributions and incorrectly awarded a separate disability exemption credit, then misclassified the renter credit as refundable. Correct AGI is $22,534.34, and the applicable $156.47 exemptions plus $60 nonrefundable renter credit reduce $223.27 to $6.80." -us,scenario_023,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model reduced earned income by the desired traditional 401(k) contribution and then estimated the CalEITC from an invented $14,665 base. The applicable adjusted-earnings input is $17,442.65, whose childless-filer CalEITC schedule yields $148.31." -us,scenario_023,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly required a dependent for CalEITC eligibility. A qualifying child is not required for the childless CalEITC schedule, and adjusted earnings of $17,442.65 produce a $148.31 refundable credit." -us,scenario_023,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model recognized the childless CalEITC pathway but did not apply its schedule consistently, first deriving roughly $44 and then submitting $358 based on nonexistent credit interactions. The childless schedule at $17,442.65 of adjusted earnings yields $148.31, with no Young Child Tax Credit." -us,scenario_023,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly identified CalEITC as the sole applicable refundable state credit but assigned $1,208 without applying the childless-filer phase-out schedule. At adjusted earnings of $17,442.65, that schedule yields $148.31." -us,scenario_023,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used a childless CalEITC cutoff below the household's earnings and therefore treated the credit as fully phased out. Adjusted earnings of $17,442.65 remain within the applicable schedule and generate $148.31." -us,scenario_023,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly treated the $8,000 taxable 403(b) distribution as disqualifying investment income for CalEITC. The household remains eligible, and the CalEITC calculation on $17,442.65 of adjusted earnings yields $148.31." -us,scenario_023,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model identified the correct childless CalEITC pathway but estimated $285 instead of applying the actual 2026 schedule. The schedule applied to $17,442.65 of adjusted earnings yields $148.31." -us,scenario_023,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly calculated CalEITC as a flat 85% of federal EITC. California uses its own earned-income schedule, which produces $148.31 at adjusted earnings of $17,442.65." -us,scenario_023,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model substituted an estimated higher-AGI phase-out calculation for California's applicable adjusted-earnings calculation. Applying the CalEITC schedule to $17,442.65 yields $148.31, not $100." -us,scenario_023,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted the household's childless CalEITC eligibility. Adjusted earnings of $17,442.65 qualify under the California schedule and generate $148.31." -us,scenario_023,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model selected the correct CalEITC program but used an unsupported approximate credit amount. The 2026 childless-filer schedule at $17,442.65 of adjusted earnings yields $148.31." -us,scenario_023,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly placed the household above the CalEITC eligibility limit. Adjusted earnings of $17,442.65 remain eligible and produce a $148.31 childless CalEITC." -us,scenario_023,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used the higher AGI figure of $22,535 to declare the childless CalEITC fully eliminated. PolicyEngine's California calculation uses adjusted earnings of $17,442.65, which fall within the credit schedule and yield $148.31." -us,scenario_023,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to recognize the childless CalEITC pathway indicated by the California residence and earnings. The household's $17,442.65 of adjusted earnings generate $148.31." -us,scenario_023,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model omitted CalEITC solely because no refundable credit was explicitly listed as an input fact. CalEITC is computed from the household facts, and $17,442.65 of adjusted earnings produce $148.31." -us,scenario_023,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model correctly isolated CalEITC and excluded the young-child and foster-youth credits, but it understated the phase-out result. The California schedule at adjusted earnings of $17,442.65 yields $148.31 rather than $114." -us,scenario_023,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model recognized the childless CalEITC but used an approximate value that overstates the scheduled credit. At $17,442.65 of adjusted earnings, the credit is $148.31." -us,scenario_023,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model applied an approximate childless CalEITC amount rather than the exact California schedule. That schedule produces $148.31 at adjusted earnings of $17,442.65, not $160." -us,scenario_023,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model omitted the refundable childless CalEITC generated by the listed California earnings. The applicable adjusted earnings are $17,442.65, producing $148.31." -us,scenario_023,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly concluded that the filer qualified for no refundable state credit. California permits a childless CalEITC here, and its schedule produces $148.31 from $17,442.65 of adjusted earnings." -us,scenario_023,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model incorrectly made federal EITC eligibility a prerequisite for CalEITC and used the federal no-child phase-out to deny the state credit. California's own CalEITC rules apply to $17,442.65 of adjusted earnings and yield $148.31." -us,scenario_023,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model used an estimated 2024 maximum, an AGI-based phase-out, and an assumed 3.25% rate instead of the 2026 California schedule. Applying the correct schedule to adjusted earnings of $17,442.65 produces $148.31." -us,scenario_023,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value for state_refundable_credits. The required output is the $148.31 CalEITC generated from $17,442.65 of adjusted earnings." -us,scenario_023,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no value for state_refundable_credits. The required output is the $148.31 CalEITC generated from $17,442.65 of adjusted earnings." -us,scenario_023,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly classified CalEITC as nonrefundable for filers without children. CalEITC is refundable on the childless-filer pathway, and $17,442.65 of adjusted earnings produces $148.31." -us,scenario_023,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model both overstated CalEITC and incorrectly added California's renter credit to refundable credits. The renter credit is nonrefundable and excluded, while the CalEITC schedule at $17,442.65 of adjusted earnings yields the entire $148.31 total." +us,scenario_023,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the generic disability flag as sufficient for SNAP's elderly-or-disabled household rules, then allowed a medical deduction and uncapped excess-shelter deduction. It also submitted $3,936 despite its own successive calculations yielding $2,352, $2,376, and $2,328, so its final value does not follow from its reasoning." +us,scenario_023,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model imposed the ordinary 130% FPL gross-income test and declared the household ineligible, overlooking categorical eligibility through TANF non-cash assistance. Rent is not a disqualifying resource; it enters the excess-shelter deduction, and the $130 asset balance passes the asset test." +us,scenario_023,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated the generic disability flag as establishing SNAP statutory disability, allowing medical expenses and an uncapped shelter deduction. It then submitted $3,768 even though its stated maximum-minus-contribution calculation yielded about $2,529." +us,scenario_023,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly unlocked the SNAP medical and uncapped shelter deductions from the generic disability flag. Its submitted $2,549 also contradicts its own computed $149 monthly benefit, which annualizes to $1,788." +us,scenario_023,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model denied eligibility under a gross-income limit and a shelter-cap calculation, omitting TANF non-cash categorical eligibility. The correct pathway proceeds to the net-income test and yields positive monthly allotments." +us,scenario_023,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model treated the generic disability fact as sufficient to authorize SNAP's medical-expense deduction and removal of the excess-shelter cap, driving net income down to $328.30. The applicable calculation produces $876.51 of net income, followed by monthly allotments of $35.20 and later $48.18 rather than a flat $194." +us,scenario_023,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income screen and stopped, overlooking categorical eligibility through TANF non-cash assistance. After the applicable deductions, the household passes the net-income test and receives a positive allotment." +us,scenario_023,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated the generic disability flag as unlocking medical and uncapped shelter deductions and overstated the medical deduction as $747.42 monthly, reducing net income to zero. The applicable deduction rules leave $876.51 in net income, so the household does not receive the maximum allotment." +us,scenario_023,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model incorrectly used the generic disability flag to remove the shelter cap and authorize a medical deduction, then added an unlisted standard utility allowance despite the instruction that unlisted facts are false or zero. Those errors reduce its net income to zero instead of $876.51." +us,scenario_023,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,The answer supplies no computation supporting a maximum annual allotment. The correct monthly calculation subtracts 30% of $876.51 from the applicable maximum allotment and aggregates changing monthly parameters to $461.34. +us,scenario_023,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated the generic disability flag as authorizing medical deductions and an uncapped excess-shelter deduction, reducing net income to zero. The applicable capped calculation leaves net income of $876.51, so the benefit is only the residual after the expected contribution." +us,scenario_023,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model incorrectly invoked uncapped shelter and medical deductions from the generic disability fact and estimated net income near $300. The applicable rules leave $876.51 of net income and therefore much smaller monthly benefits. +us,scenario_023,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model based its estimate on uncapped shelter expenses for a disabled individual, although the generic disability flag does not establish SNAP statutory disability. Applying the relevant cap leaves $876.51 in net income and an annual benefit of $461.34." +us,scenario_023,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model treated the generic disability flag as waiving the gross test and authorizing both a medical deduction and an uncapped $14,836.80 shelter deduction. Eligibility instead comes through TANF non-cash categorical eligibility, while the applicable deduction calculation leaves $876.51 monthly net income." +us,scenario_023,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model inferred ineligibility without applying TANF non-cash categorical eligibility, the passing $130 asset test, or the net-income deductions. Those steps yield $876.51 of net income and positive monthly SNAP benefits." +us,scenario_023,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of listed SNAP receipt as a reason to assign zero, despite the prompt's instruction to assume program take-up when required. PolicyEngine determines categorical eligibility through TANF non-cash assistance and computes positive benefits from $876.51 of net income." +us,scenario_023,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the generic disability flag as authorizing the medical deduction and uncapped shelter deduction, reducing countable net income to zero. The applicable capped deduction calculation leaves $876.51, so the household receives the maximum minus its 30% expected contribution rather than the maximum." +us,scenario_023,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model assumed the generic disability fact removed the excess-shelter cap and therefore converted high rent into a maximum allotment. The applicable calculation leaves $876.51 of net income and produces only the residual allotment after the expected contribution. +us,scenario_023,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,The model correctly recognized the student work-hours exception but incorrectly applied medical and uncapped shelter deductions based solely on the generic disability flag. That lowered its estimated net income far below the applicable $876.51. +us,scenario_023,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model concluded that earnings and the retirement distribution eliminate the benefit without applying TANF non-cash categorical eligibility and the relevant income deductions. The resulting net income is $876.51, below the net-income threshold, and produces a positive allotment." +us,scenario_023,snap,grok-4.3,llm_error,categorical_eligibility,False,The model asserted that income and assets exceed SNAP limits even though assets are only $130 and the household has TANF non-cash categorical eligibility. The household passes the net-income test at $876.51 and receives positive benefits. +us,scenario_023,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model treated the generic disability flag as sufficient for a medical deduction and an uncapped $14,837 shelter deduction. The applicable deduction calculation leaves $876.51 monthly net income, not the model's approximately $291 monthly figure." +us,scenario_023,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model both omitted the $8,000 taxable 403(b) distribution from income before shelter and treated the generic disability flag as authorizing medical and uncapped shelter deductions. Those errors reduce net income to zero instead of $876.51 and incorrectly produce a maximum allotment." +us,scenario_023,snap,inkling,llm_error,categorical_eligibility,False,"The model assumed an uncapped shelter deduction and medical deduction based solely on the generic disability fact, estimating net income at $300. The applicable calculation leaves $876.51 and yields much smaller residual monthly benefits." +us,scenario_023,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated the generic disability flag as exempting the household from the gross test and authorizing medical and uncapped shelter deductions, reducing net income to zero. Eligibility instead follows through TANF non-cash categorical eligibility, and the applicable deduction calculation leaves $876.51." +us,scenario_023,snap,kimi-k3,parse_contract_failure,missing_output,False,"The model provided no SNAP value or explanation, so it failed the required output contract." +us,scenario_023,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income test and stopped, overlooking TANF non-cash categorical eligibility. It also reported $22,535 of gross income instead of the listed $17,443 wages plus $8,000 taxable distribution, which total $25,443." +us,scenario_023,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $8,000 taxable 403(b) distribution from gross income and failed to carry its stated $944 net income into the benefit formula. Subtracting 30% of $944 from a $258 maximum would not yield $258, while the applicable computation uses $876.51 net income and changing monthly maximum allotments." +us,scenario_023,snap,qwen3.8-max,llm_error,categorical_eligibility,False,The model declared income and resources too high without applying TANF non-cash categorical eligibility or recognizing that bank assets are only $130. The household passes the net-income test at $876.51 and receives positive monthly benefits. +us,scenario_023,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model treated the narrative phrase “is disabled” as establishing SSI disability eligibility, despite the prompt directing that unlisted program-status inputs are false. The head fails the SSI aged, blind, or disabled gate, so applying earned-income exclusions, counting the 403(b) distribution, and subtracting income from the federal-plus-California payment standard was inapplicable." +us,scenario_023,ssi,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract." +us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model rounded the bracket tax and exemption credit too aggressively and then treated the $6.80 residual as zero. The exact calculation is $223.27 minus $156.47 and $60, leaving $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model wrongly treated CalEITC as a nonrefundable credit that offsets this output; CalEITC belongs in refundable state credits. It also omitted the taxable $8,000 distribution and retirement adjustments from its taxable-income calculation." +us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model understated the bracket tax and invoked unspecified disability considerations instead of applying the actual $156.47 exemption amount and $60 renter credit. Those credits reduce $223.27 to $6.80, not zero." +us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly identified that the personal exemption alone does not eliminate the tax, then erased the remaining liability through unsupported rounding. Applying the exact standard deduction, brackets, $156.47 exemptions, and $60 renter credit leaves $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model asserted that the roughly $150 exemption credit and other unspecified offsets eliminated the tax. The applicable nonrefundable credits total $216.47 against $223.27 of tax, leaving $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model invented a second California exemption credit solely because the taxpayer is disabled. The applicable exemption amount is $156.47 total, and with the $60 renter credit it reduces $223.27 to $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invoked nonexistent disability-related California credits and then claimed the personal exemption exceeded the tax despite its own tax estimate being larger. The applicable credits are $156.47 of exemptions plus the $60 renter credit, leaving $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $14,446 by dropping the taxable $8,000 403(b) distribution and mishandling the retirement deductions. California AGI is $22,534.34, producing $16,828.34 of taxable income and $6.80 after credits." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used approximate deduction, bracket, and exemption values that overstated the residual tax. The exact amounts are $223.27 of tax and $216.47 of credits, leaving $6.80 rather than $15." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model falsely stated that the California standard deduction exceeds AGI. AGI is $22,534.34, the deduction is $5,706, and taxable income is $16,828.34." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"The model assumed the personal exemption fully offsets the tax. Tax is $223.27, while exemptions and the renter credit total $216.47, leaving $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated taxable income as $8,846, excluding taxable income or applying unsupported deductions. California taxable income is $16,828.34, and the resulting $223.27 tax exceeds the $216.47 credits by $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The answer is consistent with reporting tentative bracket tax while failing to subtract the applicable nonrefundable exemption and renter credits. The trace subtracts $216.47 from $223.27, leaving $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,The model explicitly applied no nonrefundable state credits and also used outdated deduction and bracket parameters. The applicable $156.47 exemptions and $60 renter credit reduce the correctly computed $223.27 tax to $6.80. +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,The model incorrectly treated the available deductions and credits as fully eliminating liability. The exact nonrefundable credits are $6.80 less than the $223.27 tentative tax. +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly subtracted state amounts associated with the federal American Opportunity Credit. California instead applies $156.47 of exemptions and a $60 renter credit to $223.27 of tax, yielding $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model substituted a 2.5% early-distribution additional tax for the requested California regular income tax after nonrefundable credits. The traced output is the $223.27 regular tax less $216.47 of credits, or $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The model assumed the personal exemption and renter credit fully offset the tentative tax. They total $216.47 against $223.27, leaving $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,The model followed the correct structure but used projected and rounded parameters that understated the residual by $2.80. Exact bracket tax of $223.27 less $216.47 of credits equals $6.80. +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,The model treated the exemption and renter credits as sufficient to erase the tax. Their combined $216.47 value leaves $6.80 of the $223.27 tentative tax unpaid. +us,scenario_023,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model asserted that the standard deduction and low income produce zero tax, ignoring $16,828.34 of taxable income after the deduction. That income generates $223.27 of tax and a $6.80 liability after credits." +us,scenario_023,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used approximate projected deduction, bracket, and exemption parameters, producing a $9 residual. The exact tax is $223.27 and exact credits are $216.47, so the residual is $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,other,False,"The model subtracted the $60 renter credit but omitted the $156.47 exemption credits. Both reduce the $223.27 tentative tax, leaving $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model understated AGI by $8,088.34, effectively dropping the taxable 403(b) distribution and mishandling adjustments. Correct taxable income is $16,828.34, not roughly $8,720, and the post-credit tax is $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_023,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_023,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model wrongly added back the traditional 401(k) and IRA deductions, inflating California AGI to $25,443, and then applied no credits. AGI is $22,534.34, and $216.47 of nonrefundable credits reduces $223.27 of tax to $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model failed to deduct the traditional retirement contributions, repeatedly used the wrong filing status and bracket structure, and invented a separate disability exemption credit. The correct path uses single-filer AGI of $22,534.34 and only $156.47 of exemptions plus the $60 renter credit, leaving $6.80." +us,scenario_023,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,The model assumed the standard deduction and nonrefundable credits eliminate all California tax. They leave a $6.80 residual because $223.27 of tentative tax exceeds the $216.47 credits. +us,scenario_023,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly identified childless CalEITC but estimated the phase-out instead of applying California's schedule to adjusted earnings of $17,442.65. That computation yields $148.31, not $402." +us,scenario_023,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly required a dependent for CalEITC. A 28-year-old filer without qualifying children is eligible under the childless CalEITC pathway, which produces $148.31 at adjusted earnings of $17,442.65." +us,scenario_023,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model recognized that only childless CalEITC applies but never consistently applied its schedule, first deriving roughly $44 and then submitting $358 without a supporting calculation. The California formula applied to $17,442.65 of adjusted earnings yields $148.31." +us,scenario_023,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model called the childless CalEITC small but submitted $1,208, a value inconsistent with the childless phase-out schedule it described. Applying that schedule to adjusted earnings of $17,442.65 produces $148.31." +us,scenario_023,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model placed $17,442.65 beyond the childless CalEITC phase-out endpoint. Those earnings remain inside the eligible range and generate a $148.31 credit." +us,scenario_023,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly treated the $8,000 taxable 403(b) distribution as disqualifying investment income for CalEITC. The filer remains eligible, and the California calculation on adjusted earnings of $17,442.65 produces $148.31." +us,scenario_023,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model identified the correct childless CalEITC pathway but estimated $285 rather than applying the California credit schedule. At adjusted earnings of $17,442.65, the scheduled credit is $148.31." +us,scenario_023,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly calculated CalEITC as a flat 85% of federal EITC. California uses its own earned-income schedule, which gives $148.31 for $17,442.65 of adjusted earnings." +us,scenario_023,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model used an estimated phase-out based on a higher AGI of $22,535 instead of the traced California credit calculation based on adjusted earnings of $17,442.65. The applicable schedule produces $148.31." +us,scenario_023,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model denied all California refundable-credit eligibility despite the filer qualifying for childless CalEITC. Applying the credit formula to adjusted earnings of $17,442.65 yields $148.31." +us,scenario_023,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model selected the correct CalEITC program but approximated the amount as $180. California's schedule applied to adjusted earnings of $17,442.65 yields $148.31." +us,scenario_023,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly placed the head's income above the CalEITC eligibility limit. Adjusted earnings of $17,442.65 remain eligible and produce $148.31 under the childless schedule." +us,scenario_023,state_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used its $22,535 AGI figure to eliminate the credit instead of applying the CalEITC formula to adjusted earnings of $17,442.65. The correct income base remains within the childless schedule and yields $148.31." +us,scenario_023,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model overlooked the filer's childless CalEITC eligibility. The California schedule on adjusted earnings of $17,442.65 generates a refundable credit of $148.31." +us,scenario_023,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model overlooked the applicable childless CalEITC pathway. The filer qualifies, and adjusted earnings of $17,442.65 produce $148.31." +us,scenario_023,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model correctly isolated childless CalEITC but approximated its phase-out amount as $114. Applying California's schedule to $17,442.65 of adjusted earnings gives $148.31." +us,scenario_023,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model recognized childless CalEITC but used an inaccurate estimate of its amount. The applicable schedule at $17,442.65 of adjusted earnings produces $148.31, not $194." +us,scenario_023,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model identified the correct refundable credit but approximated the childless phase-out as $160. California's formula at adjusted earnings of $17,442.65 yields $148.31." +us,scenario_023,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model failed to generate the childless CalEITC despite the filer satisfying its eligibility conditions. The applicable calculation produces $148.31. +us,scenario_023,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable California credit applied. The filer qualifies for childless CalEITC, worth $148.31 at adjusted earnings of $17,442.65." +us,scenario_023,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model incorrectly conditioned CalEITC on qualification for federal EITC and therefore assigned zero when the federal childless credit phased out. California has its own CalEITC eligibility schedule, which awards $148.31 here." +us,scenario_023,state_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model imported an estimated federal-style maximum, phase-out threshold, and 3.25% reduction instead of using the California CalEITC schedule for 2026. Applying the state schedule to adjusted earnings of $17,442.65 yields $148.31." +us,scenario_023,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model found the correct childless CalEITC pathway but substituted a rough $300 estimate for the scheduled calculation. The California formula produces $148.31 at adjusted earnings of $17,442.65." +us,scenario_023,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required output contract." +us,scenario_023,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required output contract." +us,scenario_023,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model falsely classified CalEITC as nonrefundable for filers without children. Childless CalEITC is refundable, and this filer receives $148.31." +us,scenario_023,state_refundable_credits,qwen-3.7-max,llm_error,other,False,The model both overstated CalEITC by estimating $251 instead of applying the schedule and incorrectly added California's $60 renter's credit to refundable credits even though that credit is nonrefundable. Only the $148.31 CalEITC belongs in this output. +us,scenario_023,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model overlooked the filer's eligibility for childless CalEITC. Applying California's formula to adjusted earnings of $17,442.65 produces $148.31." us,scenario_023,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_025,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s own bracket calculation produced approximately $7,019, but it submitted $3,325 without any intervening deduction or credit. The submitted value contradicts its stated derivation." -us,scenario_025,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $29,900 standard deduction instead of the 2026 joint-filer amount of $32,200 and then applied an overstated effective tax calculation. Taxable income is $62,725.30, and the applicable brackets yield $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model approximated both the standard deduction and the 10% bracket threshold, using $32,300 and about $24,000. The exact 2026 parameters produce taxable income of $62,725.30 and tax of $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly reconstructed taxable income near $62,725 and explicitly calculated tax near $7,031, then replaced that result with $6,934 without a tax-rule adjustment. Its submitted amount contradicts its own computation." -us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $30,000 standard deduction instead of the 2026 married-joint amount of $32,200, leaving taxable income overstated by about $2,200. It also used 2025 bracket thresholds rather than the controlling 2026 thresholds." -us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model explicitly calculated $7,031 from the correct income, standard deduction, and bracket structure, then submitted $4,700 despite identifying no nonrefundable credit. The final value is disconnected from its stated derivation." -us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used the $30,000 2025 joint standard deduction rather than the $32,200 deduction for 2026. That inflated taxable income to $64,925 and produced $7,314 instead of applying the 2026 deduction and brackets." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the listed employer-sponsored insurance premium from wages, invented mortgage interest and state taxes from balances, and restored personal exemptions. The trace includes the full $62,725.29 of wages in AGI and uses the $32,200 standard deduction because itemized deductions are only $2,354.43." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used a $22,400 standard deduction and therefore computed taxable income of about $72,525. The 2026 married-joint standard deduction is $32,200, leaving taxable income of $62,725.30." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a post-TCJA-expiration baseline with restored personal exemptions and reverted parameters. The controlling 2026 law retains the $32,200 joint standard deduction and the applicable 10% and 12% brackets, yielding $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assumed TCJA expiration, used a $16,100 standard deduction plus personal exemptions, and taxed the upper portion at 15%. For 2026 the household instead receives the $32,200 joint standard deduction and its taxable income remains within the 12% bracket after the initial 10% band." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The answer implies use of the wrong 2026 deduction or rate schedule because the stated $94,925 AGI leads to $62,725.30 of taxable income under the $32,200 joint standard deduction. Applying the controlling brackets to that amount yields $7,031.04, not $8,993." -us,scenario_025,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model estimated the joint standard deduction at $30,900 instead of using $32,200. This overstated taxable income by about $1,300 and consequently overstated tax." -us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model erased the positive liability through unspecified deductions or credits even though it identified wages and pension income. After the $32,200 standard deduction, $62,725.30 remains taxable, and no nonrefundable credit reduces the resulting $7,031.04 tax." -us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $2,460 answer is consistent with taxing only the initial 10% bracket and omitting the 12% tax on the remaining taxable income. Taxable income is $62,725.30, so income above the 10% threshold also generates tax." -us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that deductions and credits eliminate all $94,925.30 of AGI. The $32,200 standard deduction leaves $62,725.30 taxable, and no applicable nonrefundable credits eliminate the resulting liability." -us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied an expired-law regime with a reduced standard deduction, restored personal exemptions, and a 15% bracket. The controlling 2026 schedule uses the $32,200 joint standard deduction and taxes the remaining $62,725.30 through the 10% and 12% bands." -us,scenario_025,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed reverted 2026 law, combining a $16,800 standard deduction with personal exemptions and applying a 15% marginal rate. The actual computation uses a $32,200 standard deduction, no personal exemptions, and the applicable 10% and 12% joint brackets." -us,scenario_025,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. It therefore failed the required submission contract before any substantive tax computation could be evaluated. -us,scenario_025,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model acknowledged taxable income around $63,425 and a positive pre-credit tax of roughly $5,500–$6,500, then submitted zero despite identifying no nonrefundable credit. A positive bracket calculation cannot round to zero, and the exact calculation yields $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used a $31,450 joint standard deduction rather than $32,200, overstating taxable income by $750. It also used estimated bracket thresholds instead of the exact 2026 parameters, producing $7,130 rather than $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own calculation produced approximately $7,019, but it submitted $3,325 without any deduction or credit supporting that reduction. It violated the answer contract by replacing its reasoned result with an unrelated final value." +us,scenario_025,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $29,900 standard deduction instead of the 2026 married-filing-jointly amount of $32,200, then misapplied the brackets: even its stated $65,025 taxable income would not produce $8,640 under the applicable 10% and 12% rates. The correct $62,725.30 taxable income produces $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model approximated both the standard deduction and the 10% bracket threshold rather than applying the exact 2026 parameters. The exact $32,200 standard deduction and applicable brackets yield $7,031.04, not $7,035." +us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived taxable income near $62,725 and explicitly calculated tax near $7,031, but then submitted $6,934 with no supporting computation. Its final value contradicts its own correct tax calculation." +us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the obsolete $30,000 standard deduction and 2025 bracket thresholds after speculating about whether current law would continue. For 2026 the married-filing-jointly standard deduction is $32,200, leaving $62,725.30 taxable and $7,031.04 of tax." +us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly calculated approximately $7,031 from the $32,200 standard deduction and 2026 brackets, then submitted $4,700 without identifying any nonrefundable credit. With no such credit, the calculated $7,031.04 remains the requested output." +us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $30,000 standard deduction and 2025 married-filing-jointly brackets instead of the 2026 parameters. The $32,200 deduction leaves $62,725.30 taxable, to which the 2026 brackets apply for $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the stated employer-sponsored insurance premium from wages, invented mortgage interest and state taxes from a mortgage balance and unspecified facts, and applied personal exemptions. Those inputs do not reduce AGI or taxable income here; the household instead takes the $32,200 standard deduction and owes $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used a $22,400 married-filing-jointly standard deduction instead of $32,200, so its asserted taxable income was overstated. It also submitted $5,236, which is incompatible with its own stated taxable income under the applicable brackets; the correct taxable income is $62,725.30 and the tax is $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a post-TCJA-expiration baseline with personal exemptions rather than the applicable 2026 current-law standard deduction and rate schedule. Using the $32,200 standard deduction and 2026 married-filing-jointly brackets yields $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration, used a $16,100 standard deduction plus personal exemptions, and applied a restored 15% bracket. The applicable 2026 rules use a $32,200 standard deduction and the 10%/12% schedule over this income, producing $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted $8,993 implies the wrong 2026 deduction or rate schedule despite correctly identifying $94,925 of AGI. Applying the $32,200 married-filing-jointly standard deduction and the applicable brackets to $62,725.30 yields $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model substituted estimated values of $30,900 for the standard deduction and $24,600 for the first bracket ceiling. The exact 2026 parameters, including the $32,200 standard deduction, produce $62,725.30 of taxable income and $7,031.04 of tax." +us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that deductions and unspecified credits eliminate the tax despite $94,925.30 of AGI and no applicable nonrefundable credits. The $32,200 standard deduction leaves $62,725.30 taxable, producing $7,031.04 rather than zero." +us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $2,460 answer captures only about the 10% bracket tax and omits the 12% tax on the remainder of $62,725.30 of taxable income. Applying both bracket segments yields $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model falsely stated that deductions or credits eliminate all taxable income from $94,925.30 of wages and pension. The standard deduction is $32,200, no nonrefundable credit applies, and the remaining $62,725.30 generates $7,031.04 of tax." +us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a post-TCJA-expiration standard deduction, personal exemptions, and a restored 15% bracket. The applicable 2026 current-law calculation uses the $32,200 standard deduction and 10%/12% brackets, yielding $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a reverted $16,800 standard deduction, personal exemptions, and a 15% bracket rather than the applicable 2026 rules. The $32,200 standard deduction leaves $62,725.30 taxable and produces $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested variable. It therefore failed the required structured-output contract without performing a usable tax computation. +us,scenario_025,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model recognized that the household had roughly $63,425 of taxable income and a positive tax liability, but then set the result to zero despite identifying no nonrefundable credit. The correct taxable income is $62,725.30 and, with no applicable credit, the tax remains $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used estimated 2026 parameters—a $31,450 standard deduction and $24,350 first-bracket ceiling—instead of the exact values. The $32,200 standard deduction and applicable bracket thresholds produce $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model used a fictitious $9,300 married-filing-jointly standard deduction, separately deducted medical expenses despite the standard deduction being larger, and treated a senior deduction as a tax credit. The 61-year-old head does not qualify for an age-65 additional deduction, and the household takes the $32,200 standard deduction, yielding $7,031.04." us,scenario_025,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_025,head_medicare_eligible,gpt-5.4-nano,llm_error,other,False,The model's reasoning correctly states that the 61-year-old head is not Medicare eligible because traditional Medicare begins at age 65 and no disability or ESRD pathway is provided. It then violated the answer contract by submitting value = 1 for a yes result despite its own explanation requiring value = 0. -us,scenario_025,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly identified the spouse's $62,725 wages as the FICA base, then added a nonexistent Ohio employee payroll tax by treating Ohio income tax as part of payroll tax. Its final $9,432.76 also contradicts its own component arithmetic, inflating the correct FICA-only total with unsupported state and adjustment amounts." -us,scenario_025,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model applied the correct payroll-tax base and rules, but submitted $4,800.14 instead of the $4,798.46 amount it computed from 6.2% Social Security tax and 1.45% Medicare tax on $62,725. The error is an unsupported final-value substitution labeled as rounding rather than a rule error." -us,scenario_025,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages by the head's $21,208 employer-sponsored insurance premiums, using $41,517 instead of the listed $62,725 of spouse wages. Employee Social Security and Medicare taxes apply to the FICA wage base here, and the model's deduction removed wages that PolicyEngine taxes." -us,scenario_025,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct rule, 7.65% employee FICA on the spouse's $62,725 wages with no Additional Medicare Tax, but submitted $9,571.50 instead of the resulting $4,798.48. Its numeric answer is inconsistent with its own explanation and reflects an arithmetic or final-output transcription error." -us,scenario_025,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model said it computed payroll tax on annual wages for both spouses, but only the spouse has listed gross wages subject to employee FICA. Its $9,106 answer is consistent with expanding the payroll-tax base beyond the listed $62,725 of wages, such as by inferring additional spousal earnings or including nonwage income." -us,scenario_025,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable payroll_tax value or supporting explanation. This is a missing-output failure rather than a substantive payroll-tax computation. -us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly identified the $200 retirement credit but used an oversimplified 2.75% calculation and then introduced an unsupported downward adjustment to $875. The 2026 schedule produces $2,121.57 on $91,125.30 before the retirement credit, leaving $1,921.57." -us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,The model incorrectly started Ohio taxable income from federal taxable income after the federal standard deduction; Ohio starts from federal AGI here and then subtracts Ohio personal exemptions. It also used obsolete rates up to 5.75% and omitted the $200 retirement-income credit. -us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used $1,850 rather than $1,900 for each personal exemption and explicitly failed to apply the available $200 retirement-income credit. It also estimated the bracket result instead of applying the 2026 schedule that yields $2,121.57 before credits." -us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model reached the correct exemption deduction and identified the $200 retirement credit, but its submitted $2,426 contradicts its own $1,790 tax calculation and stated credits. Applying the 2026 schedule to $91,125.30 yields $2,121.57, followed by the $200 credit." -us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used unsupported $2,500 personal exemptions and an obsolete 2.75%-above-threshold shortcut. The correct exemptions total $3,800, and the 2026 schedule produces $2,121.57 before subtracting the $200 retirement credit." -us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $2,400 rather than $1,900 per personal exemption and applied the obsolete 2.75%-above-$26,050 shortcut. Although it correctly subtracted the $200 retirement credit, the correct pre-credit tax is $2,121.57." -us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model invented a senior credit and treated the retirement credit as $250 rather than $200, despite neither spouse qualifying for an additional age-based credit on these facts. It also used the wrong personal exemptions and rate schedule." -us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied an obsolete multi-rate schedule of 2.75%, 3.226%, and 3.688%, used only one $2,400 exemption, and omitted the $200 retirement credit. The correct taxable income is $91,125.30 and its scheduled tax is $2,121.57 before that credit." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the listed $21,208 employer-sponsored insurance premium from wages even though gross wages were supplied as the annual taxable input, then invented an Ohio medical-expense deduction. Those reductions do not alter the traced Ohio AGI of $94,925.30; only $3,800 of personal exemptions applies before the tax schedule and $200 retirement credit." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $1,695 estimate does not implement the traced Ohio calculation. Taxable income of $91,125.30 produces $2,121.57 under the 2026 schedule, and the $200 retirement credit leaves $1,921.57." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model used a 2.75%-above-threshold shortcut and stopped at its estimated tax without applying the $200 nonrefundable retirement-income credit. The actual scheduled tax before that credit is $2,121.57." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used unsupported $2,700 personal exemptions instead of $1,900 each and applied an obsolete 2.75% shortcut. It also omitted the $200 retirement-income credit generated by the $32,200 taxable pension." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly derived $91,125 of taxable income and applied the $200 retirement credit, but calculated the pre-credit tax as a flat 2.75% of income above $26,050. The 2026 Ohio schedule instead produces $2,121.57 before the credit." -us,scenario_025,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the federal standard deduction when calculating Ohio taxable income and used $2,400 rather than $1,900 personal exemptions. Ohio taxable income is $91,125.30, not $60,125, and the $200 retirement credit must be subtracted after computing tax." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented a low-income or family-based offset that does not apply to this $94,925.30 Ohio AGI household. The schedule produces $2,121.57 before credits, and the sole traced nonrefundable retirement credit reduces it only to $1,921.57." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $1,340 estimate reflects an unspecified standard-deduction and rate shortcut rather than Ohio's AGI-based computation. Ohio taxable income is $91,125.30, scheduled tax is $2,121.57, and the retirement credit is $200." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model invented a $230.625 medical-expense deduction from Ohio income. No such reduction appears in this computation: $94,925.30 less $3,800 of personal exemptions gives $91,125.30, followed by $2,121.57 of tax and a $200 retirement credit." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model used $2,400 rather than $1,900 per exemption, applied the obsolete 2.75%-above-threshold shortcut, and explicitly omitted the available $200 retirement-income credit. Each of those steps lowers the result incorrectly." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly used two $1,900 exemptions and the $200 retirement credit but computed the scheduled tax as $2,150.25 instead of $2,121.57. The correct post-credit amount is therefore $1,921.57." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model wrongly excluded the entire $32,200 private pension from Ohio taxable income even though it is included in Ohio AGI. Ohio instead taxes it in AGI and provides a $200 nonrefundable retirement-income credit, not a $25 credit or wholesale pension exclusion." -us,scenario_025,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and credits reduce taxable income to zero without identifying any applicable provision. Ohio AGI remains $94,925.30, and only $3,800 of personal exemptions and a $200 retirement credit reduce the resulting liability." -us,scenario_025,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used $2,400 rather than $1,900 personal exemptions, applied a 2.75%-above-$26,050 shortcut, and omitted the $200 retirement credit. The correct schedule produces $2,121.57 on $91,125.30 before that credit." -us,scenario_025,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used unsupported $2,650 exemptions and an obsolete tiered schedule with a 3.5% band beginning at $46,100. It also omitted the $200 retirement-income credit, which must reduce the correctly scheduled $2,121.57 tax." -us,scenario_025,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,credit_phaseout,False,"The model used $2,500 rather than $1,900 per exemption, relied on the obsolete 2.75%-above-threshold shortcut, and omitted the $200 retirement-income credit. The correct taxable income is $91,125.30 and the final tax is $1,921.57." -us,scenario_025,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly derived $91,125 of taxable income and subtracted the $200 retirement credit, but used a flat 2.75% calculation above $26,050. The 2026 schedule produces $2,121.57 before the credit, not $1,789.56." -us,scenario_025,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"The model invented senior or age-based adjustments that reduce the spouse's income nearly to zero, even though the traced computation contains no such adjustment. The household instead receives two $1,900 personal exemptions and a $200 retirement-income credit." -us,scenario_025,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model included the correct wage and pension income but used an unspecified bracket estimate that overstated the scheduled tax and did not account correctly for the traced deductions and credit. The exact sequence is $94,925.30 of AGI, $3,800 of exemptions, $2,121.57 of tax, and a $200 retirement credit." -us,scenario_026,child1_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated income below its asserted CHIP ceiling as sufficient for CHIP eligibility and omitted the prerequisite that the child not qualify for Medicaid. Child 1 is Medicaid-eligible under the OLDER_CHILD category, which makes CHIP eligibility false." -us,scenario_026,child1_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model applied only an NC Health Choice income-threshold test, using its calculated $61,093 income, and failed to evaluate Medicaid eligibility first. Child 1 qualifies for Medicaid under the OLDER_CHILD category and therefore cannot qualify for CHIP." -us,scenario_026,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer or explanation for child1_chip_eligible. The required derivation is Medicaid eligibility under the OLDER_CHILD category followed by CHIP ineligibility because CHIP excludes children who qualify for Medicaid. -us,scenario_026,child1_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,The model substituted the approximately 138% FPL adult expansion limit for North Carolina's OLDER_CHILD Medicaid limit. The applicable older-child threshold admits the engine-calculated MAGI of 2.13 times FPL. -us,scenario_026,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,The model used an incorrect poverty guideline and a generic 200% FPL ceiling instead of North Carolina's OLDER_CHILD Medicaid threshold. It also wrongly treated employer-sponsored insurance as evidence against Medicaid eligibility. -us,scenario_026,child1_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied the approximately 138% FPL adult Medicaid limit to an 11-year-old. North Carolina evaluates this dependent under the OLDER_CHILD category, whose limit covers MAGI at 2.13 times FPL." -us,scenario_026,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,The model applied the approximately 138% FPL adult Medicaid limit instead of the OLDER_CHILD limit for ages 6 through 18. The correct category accepts the child's 2.13-times-FPL MAGI. -us,scenario_026,child1_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model overstated the relevant income level as approximately 260% FPL. PolicyEngine's MAGI computation yields 2.13 times FPL, which falls within North Carolina's OLDER_CHILD Medicaid limit." -us,scenario_026,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,The model invented an uninsured-status requirement for Medicaid and treated employer-sponsored insurance as disqualifying. Existing employer-sponsored coverage does not prevent this 11-year-old from qualifying through North Carolina's income-tested OLDER_CHILD pathway. -us,scenario_026,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model added the listed FLSA overtime premium to gross wages even though gross wages already include all overtime under the prompt. That double counting displaced the engine's MAGI result of 2.13 times FPL, which satisfies the OLDER_CHILD limit." -us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,The model used the 133% FPL adult threshold for an 11-year-old dependent. North Carolina's OLDER_CHILD category has a higher applicable limit that includes MAGI at 2.13 times FPL. -us,scenario_026,child1_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model imposed an incorrect 147% FPL limit on the OLDER_CHILD category. North Carolina's applicable older-child threshold permits eligibility at the traced MAGI of 2.13 times FPL. -us,scenario_026,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies an income ceiling below the child's traced MAGI of 2.13 times FPL. North Carolina's OLDER_CHILD limit covers that income level, so the income test is satisfied." -us,scenario_026,child1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied the 133% FPL adult Medicaid ceiling to the older-child pathway and diverted the child to CHIP. At 2.13 times FPL, the 11-year-old remains within North Carolina's OLDER_CHILD Medicaid limit." -us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,The model treated 133% FPL as the Medicaid ceiling for a child aged 11. The governing OLDER_CHILD threshold is higher and admits the engine-calculated MAGI of 2.13 times FPL. -us,scenario_026,child1_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,The model placed the household above North Carolina's child Medicaid limit without applying the OLDER_CHILD threshold. That threshold covers the traced MAGI of 2.13 times FPL. -us,scenario_026,child1_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,The model treated employer-sponsored insurance as a Medicaid disqualification. Coverage status does not defeat eligibility through North Carolina's OLDER_CHILD category when its categorical and MAGI tests are met. -us,scenario_026,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,The model used an unspecified benchmark ceiling below 2.13 times FPL instead of North Carolina's OLDER_CHILD limit. The applicable limit includes this 11-year-old at the traced MAGI level. -us,scenario_026,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model required an unspecified special status and failed to assign the 11-year-old dependent to the ordinary OLDER_CHILD Medicaid pathway. That category requires no SSI or disability status and its income test is met at 2.13 times FPL. -us,scenario_026,child1_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,The model placed a five-person household at 2.13 times FPL above the school-age Medicaid ceiling. North Carolina's OLDER_CHILD threshold covers that income level. -us,scenario_026,child1_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model estimated a North Carolina child Medicaid threshold below the applicable OLDER_CHILD limit. The correct limit admits MAGI at 2.13 times FPL. -us,scenario_026,child1_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,The model applied an income limit that excludes 2.13 times FPL for an 11-year-old. North Carolina's OLDER_CHILD Medicaid limit includes that MAGI level. -us,scenario_026,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model placed the household above the child Medicaid ceiling without using the OLDER_CHILD category's actual limit. The traced 2.13-times-FPL MAGI satisfies that limit. -us,scenario_026,child1_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,The model used an income ceiling below North Carolina's OLDER_CHILD limit and compounded that error by treating employer-sponsored insurance as disqualifying. The child passes the category's MAGI test at 2.13 times FPL regardless of existing employer coverage. -us,scenario_026,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,The model used the 133% FPL adult Medicaid limit for a child aged 11. The applicable OLDER_CHILD threshold is higher and covers MAGI at 2.13 times FPL. -us,scenario_026,child1_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model substituted the approximately 133% FPL adult base limit for North Carolina's OLDER_CHILD threshold. It also overstated the relevant income ratio; the engine computes 2.13 times FPL, within the applicable child limit." -us,scenario_026,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for child1_medicaid_eligible. It therefore failed the required output contract rather than performing the OLDER_CHILD eligibility computation. -us,scenario_026,child1_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model applied the 133% FPL adult limit, with a 5% disregard, to an 11-year-old dependent. North Carolina's OLDER_CHILD category uses a higher limit that includes the traced MAGI of 2.13 times FPL." -us,scenario_026,child1_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model asserted that income exceeded the Medicaid-for-children limit without applying North Carolina's OLDER_CHILD threshold. At 2.13 times FPL, the child satisfies that threshold." -us,scenario_026,child1_medicaid_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model correctly placed the stated income within its child Medicaid range, then wrongly reversed eligibility because the child has employer-sponsored insurance. Employer-sponsored coverage does not disqualify the child from North Carolina's OLDER_CHILD Medicaid pathway." -us,scenario_026,child2_chip_eligible,gemini-3.1-pro-preview,llm_error,health_coverage,False,"The model applied North Carolina's CHIP income ceiling as a sufficient eligibility test and stopped after comparing its $82,301 MAGI estimate with 216% of FPL. It omitted the prior Medicaid screen: Child 2 qualifies for Medicaid under the OLDER_CHILD category and therefore is ineligible for CHIP." -us,scenario_026,child2_chip_eligible,gemini-3.5-flash,llm_error,health_coverage,False,"The model treated income below its stated 211% FPL CHIP threshold as establishing eligibility. It failed to apply the Medicaid-exclusion step: Child 2 is Medicaid-eligible under the OLDER_CHILD category, which bars CHIP eligibility." -us,scenario_026,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer for child2_chip_eligible, violating the required structured-output contract. The required value is 0 because Child 2 qualifies for Medicaid under the OLDER_CHILD category and Medicaid eligibility excludes CHIP eligibility." -us,scenario_026,child2_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model applied an older-child Medicaid ceiling of about 211% FPL and independently estimated the household at 219% FPL. The engine places the household at 2.13 times FPL and within North Carolina's OLDER_CHILD limit, so the model used the wrong threshold and FPL/MAGI calculation." -us,scenario_026,child2_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,The model incorrectly treated employer-sponsored insurance as an additional bar to Medicaid and also asserted income above the child limit without calculating the applicable category. Child 2 qualifies through North Carolina's OLDER_CHILD pathway at 2.13 times FPL despite having employer-sponsored coverage. -us,scenario_026,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model alternated among 215%, 211%, and 133% FPL limits instead of applying the engine's 2026 North Carolina OLDER_CHILD threshold. At the traced MAGI of 2.13 times FPL, Child 2 is inside that category's limit." -us,scenario_026,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model treated 138% FPL as the Medicaid ceiling for an 11-year-old and treated the higher child limit as CHIP-only. PolicyEngine assigns Child 2 to the OLDER_CHILD Medicaid category, whose applicable limit includes a MAGI of 2.13 times FPL." -us,scenario_026,child2_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,The model overstated the household income level as roughly 260% FPL and then compared it with the wrong child limit. The traced MAGI is 2.13 times FPL and falls within North Carolina's OLDER_CHILD Medicaid threshold. -us,scenario_026,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model invented an uninsured or uncovered requirement for Medicaid and disqualified Child 2 solely because employer-sponsored insurance was listed. Existing employer-sponsored coverage does not block the OLDER_CHILD Medicaid pathway, and the child meets its income test at 2.13 times FPL." -us,scenario_026,child2_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model added the separately listed FLSA overtime premium to gross wages even though the prompt states that gross wages already include overtime, thereby double-counting earnings. It also used an erroneous child-income ceiling; the resulting engine MAGI is 2.13 times FPL and satisfies the OLDER_CHILD limit." -us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used 133% FPL as North Carolina's Medicaid limit for this 11-year-old. The applicable engine category is OLDER_CHILD, and its 2026 limit admits the traced MAGI of 2.13 times FPL." -us,scenario_026,child2_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model compared its $61,093 MAGI estimate with a 133% FPL ceiling. Child 2 instead qualifies under North Carolina's OLDER_CHILD Medicaid category, whose applicable limit includes the engine-calculated 2.13-times-FPL MAGI." -us,scenario_026,child2_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model's unexplained income-above-threshold conclusion applies a ceiling below the applicable North Carolina OLDER_CHILD limit. The engine calculates MAGI at 2.13 times FPL, which passes that category's income test." -us,scenario_026,child2_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used 133% FPL as the Medicaid ceiling and diverted the child to CHIP. PolicyEngine applies the OLDER_CHILD Medicaid category, under which 2.13 times FPL remains eligible." -us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,The model applied a 133% FPL Medicaid threshold to Child 2. The applicable North Carolina OLDER_CHILD category has a higher limit that includes the traced MAGI of 2.13 times FPL. -us,scenario_026,child2_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,The model's bare assertion that household income exceeds the limit applies the wrong income ceiling. Child 2's engine-calculated MAGI is 2.13 times FPL and falls within the OLDER_CHILD threshold. -us,scenario_026,child2_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,The model treated 133% FPL as the older-child Medicaid ceiling and 210% FPL as a separate CHIP ceiling. PolicyEngine's North Carolina OLDER_CHILD Medicaid test accepts the traced 2.13-times-FPL income level; employer-sponsored insurance does not reverse that result. -us,scenario_026,child2_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model used an unspecified benchmark Medicaid threshold below the applicable OLDER_CHILD limit. At 2.13 times FPL, Child 2 satisfies North Carolina's income test for that category." -us,scenario_026,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required an unspecified special status and failed to place the 11-year-old dependent in the ordinary OLDER_CHILD Medicaid category. That category applies based on age and dependency, and its income test is met at 2.13 times FPL." -us,scenario_026,child2_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model concluded that $85,209 less limited deductions was above the North Carolina child Medicaid limit without applying the OLDER_CHILD threshold used for 2026. The engine's MAGI result is 2.13 times FPL, which falls within that limit." -us,scenario_026,child2_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model estimated a North Carolina child threshold that was too low. The applicable OLDER_CHILD limit includes Child 2's traced MAGI of 2.13 times FPL. -us,scenario_026,child2_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model compared household MAGI with the wrong Medicaid income limit for an 11-year-old. Under North Carolina's OLDER_CHILD category, the engine-calculated 2.13-times-FPL MAGI passes the income test." -us,scenario_026,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model's income-above-limit conclusion uses a ceiling below North Carolina's applicable OLDER_CHILD threshold. Child 2 remains within that threshold at 2.13 times FPL. -us,scenario_026,child2_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model applied the wrong income threshold and also treated employer-sponsored insurance as supporting ineligibility. Child 2 meets the OLDER_CHILD income limit at 2.13 times FPL, and existing employer coverage is not a Medicaid disqualifier." -us,scenario_026,child2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,The model applied a 133% FPL ceiling to Medicaid for children ages 6 through 18. PolicyEngine's applicable North Carolina OLDER_CHILD threshold includes a MAGI of 2.13 times FPL. -us,scenario_026,child2_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,The model both overstated the household at roughly 240% FPL and used a 133% FPL base limit. The traced MAGI is 2.13 times FPL and qualifies under North Carolina's OLDER_CHILD category. -us,scenario_026,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer for child2_medicaid_eligible, violating the required outputs contract. The required numeric output was 1 because the OLDER_CHILD income test is satisfied at 2.13 times FPL." -us,scenario_026,child2_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model treated 133% or 138% FPL as the ceiling for Child 2 and overstated the household MAGI as roughly 226% FPL. The engine calculates 2.13 times FPL and applies the higher OLDER_CHILD limit, which the child meets." -us,scenario_026,child2_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,The model asserted that income was well above the child limit without applying North Carolina's OLDER_CHILD threshold. The traced income level is 2.13 times FPL and remains within that category's limit. -us,scenario_026,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used 133% FPL as the Medicaid limit for an 11-year-old and also used an incorrect five-person FPL amount. PolicyEngine applies the North Carolina OLDER_CHILD category, under which the traced MAGI of 2.13 times FPL is eligible; employer-sponsored insurance does not bar eligibility." -us,scenario_026,child3_chip_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model applied the CHIP income band as sufficient for eligibility and omitted the prerequisite that the child not qualify for Medicaid. Child 3 is Medicaid-eligible under the OLDER_CHILD category and therefore ineligible for CHIP. -us,scenario_026,child3_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model compared MAGI with a CHIP limit but never applied the Medicaid-exclusion step. Child 3's OLDER_CHILD Medicaid eligibility automatically prevents CHIP eligibility. -us,scenario_026,child3_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model treated income below 211% FPL as sufficient for North Carolina CHIP eligibility. It omitted that Child 3 already qualifies for Medicaid under the OLDER_CHILD category, which disqualifies the child from CHIP." -us,scenario_026,child3_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model checked age, Medicare status, and the CHIP income range but failed to check Medicaid eligibility. Child 3 is Medicaid-eligible under the OLDER_CHILD category, so the child cannot qualify for CHIP." -us,scenario_026,child3_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model substituted a 138% FPL adult-style limit for North Carolina’s OLDER_CHILD threshold and therefore incorrectly diverted the child to CHIP. At the traced MAGI of 2.13 times FPL, the applicable older-child category yields Medicaid eligibility." -us,scenario_026,child3_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model applied an incorrect 207% FPL cutoff and understated the corresponding income limit. North Carolina’s applicable OLDER_CHILD threshold admits Child 3 at the traced MAGI of 2.13 times FPL, and employer-sponsored insurance affects coverage status rather than this eligibility result." -us,scenario_026,child3_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,The model used an incorrect approximate 211% FPL limit and compared it directly with gross wages instead of the engine’s MAGI-to-FPL result. The applicable OLDER_CHILD test admits the traced 2.13-times-FPL level. -us,scenario_026,child3_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,The model incorrectly imposed the 133% FPL limit on North Carolina’s OLDER_CHILD category and then treated CHIP as the only higher-income pathway. Child 3 qualifies under the applicable older-child Medicaid threshold at 2.13 times FPL. -us,scenario_026,child3_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model overstated the household’s income level as roughly 260% FPL and rejected eligibility without applying the traced OLDER_CHILD test. PolicyEngine calculates MAGI at 2.13 times FPL, which is within the applicable North Carolina threshold." -us,scenario_026,child3_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model incorrectly treated employer-sponsored insurance as a bar to Medicaid eligibility and also used an income threshold that excludes the traced result. Child 3’s existing coverage does not negate eligibility, and the OLDER_CHILD category admits MAGI at 2.13 times FPL." -us,scenario_026,child3_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the separately listed FLSA overtime premium by adding it to gross wages even though gross wages already include overtime, then applied a 138% FPL cutoff. The engine’s MAGI is 2.13 times FPL, and the applicable OLDER_CHILD threshold grants eligibility." -us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,The model applied a 133% FPL limit instead of North Carolina’s applicable OLDER_CHILD threshold. That category admits Child 3 at the traced MAGI of 2.13 times FPL. -us,scenario_026,child3_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model used a 133% FPL ceiling for the age-9 child and rejected eligibility on that basis. The applicable North Carolina OLDER_CHILD threshold admits the traced MAGI level of 2.13 times FPL. -us,scenario_026,child3_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies an income cutoff below the applicable North Carolina OLDER_CHILD limit. Child 3’s traced MAGI is 2.13 times FPL, which satisfies that category’s income test." -us,scenario_026,child3_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model substituted a 133% FPL ceiling for the applicable OLDER_CHILD Medicaid threshold and incorrectly assigned the child only to CHIP. At 2.13 times FPL, Child 3 passes the older-child Medicaid income test." -us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,The model incorrectly treated 133% FPL as the controlling North Carolina threshold for Child 3. The applicable OLDER_CHILD category covers the engine-calculated MAGI level of 2.13 times FPL. -us,scenario_026,child3_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model’s answer implies a North Carolina child Medicaid ceiling below 2.13 times FPL. The applicable OLDER_CHILD threshold admits that traced income level, so the income test is satisfied." -us,scenario_026,child3_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model applied a 133% FPL Medicaid ceiling and treated the approximately 210% CHIP figure as an upper bound on Medicaid eligibility. North Carolina’s applicable OLDER_CHILD category admits the traced 2.13-times-FPL MAGI, and employer-sponsored coverage does not defeat eligibility." -us,scenario_026,child3_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,The model invoked a generic benchmark threshold instead of the North Carolina OLDER_CHILD income limit. Child 3 is age 9 and qualifies at the traced MAGI of 2.13 times FPL. -us,scenario_026,child3_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model required an unspecified special status and failed to place the age-9 dependent in the OLDER_CHILD MAGI category. That category’s North Carolina income test admits the traced 2.13-times-FPL MAGI. -us,scenario_026,child3_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,The model compared earnings with an understated North Carolina child Medicaid limit rather than applying the OLDER_CHILD threshold to the engine’s MAGI-to-FPL measure. The traced 2.13-times-FPL level passes that test. -us,scenario_026,child3_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model assumed the North Carolina child Medicaid threshold was below the household’s traced income level. The applicable OLDER_CHILD threshold admits MAGI at 2.13 times FPL. -us,scenario_026,child3_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,The model applied an income limit that incorrectly excludes a 9-year-old at 2.13 times FPL. North Carolina’s applicable OLDER_CHILD threshold grants eligibility at that traced level. -us,scenario_026,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model’s conclusion rests on an understated North Carolina child Medicaid limit. Child 3’s 2.13-times-FPL MAGI is within the applicable OLDER_CHILD threshold. -us,scenario_026,child3_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"The model both understated the applicable OLDER_CHILD income threshold and treated employer-sponsored insurance as adverse to Medicaid eligibility. The traced 2.13-times-FPL MAGI qualifies, and existing employer coverage does not negate eligibility." -us,scenario_026,child3_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,The model incorrectly used 133% FPL as North Carolina’s Medicaid ceiling for children ages 6 through 18. The applicable OLDER_CHILD threshold admits Child 3 at 2.13 times FPL. -us,scenario_026,child3_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an incorrect 133% FPL base limit and overstated the household’s income as roughly 240% FPL. The engine places MAGI at 2.13 times FPL, within the applicable OLDER_CHILD threshold." -us,scenario_026,child3_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child3_medicaid_eligible, violating the required structured-output contract." -us,scenario_026,child3_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model imposed a 133% or 138% FPL ceiling and overstated MAGI as roughly 226% FPL. The traced MAGI is 2.13 times FPL, and North Carolina’s applicable OLDER_CHILD threshold admits it." -us,scenario_026,child3_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,The model assumed the household was above North Carolina’s Medicaid-for-children limit without applying the OLDER_CHILD category threshold. Child 3’s traced MAGI of 2.13 times FPL satisfies that test. -us,scenario_026,child3_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model applied a 133% FPL ceiling, used an incorrect poverty guideline, and treated employer-sponsored insurance as supporting ineligibility. North Carolina’s applicable OLDER_CHILD threshold admits the engine-calculated 2.13-times-FPL MAGI, regardless of existing employer coverage." -us,scenario_026,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model omitted the $7,746.26 overtime deduction and restricted the usable nonrefundable CTC to roughly $1,500 instead of applying the credit up to the full $4,586.51 tentative liability. The $6,600 total CTC eliminates that liability, yielding $0." -us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model explicitly derived that the CTC absorbs all tentative tax and then submitted $4,107, contradicting its own computation. It also omitted the overtime deduction, but even its overstated tentative tax was fully covered by its stated $6,000 CTC." -us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model incorrectly treated only about $605 of the CTC as nonrefundable by subtracting assumed refundable portions first. Nonrefundable CTC is applied against tentative tax up to the available $6,600 credit, so it offsets the full $4,586.51 after the overtime deduction." -us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly capped the nonrefundable CTC through a $500-per-child split and failed to apply the remaining available CTC against tax. The $6,600 credit offsets the entire $4,586.51 tentative liability; the refundable limit does not force a residual tax balance." -us,scenario_026,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $7,746 overtime premium by adding it to gross wages even though gross wages already include overtime, then failed to deduct that overtime amount under the 2026 overtime deduction. It also produced $4,200 after its own calculation yielded $446, while the correct taxable-income computation and $6,600 CTC reduce the requested liability to $0." -us,scenario_026,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied an obsolete pre-TCJA structure with personal exemptions, a 15% bracket, a $1,000-per-child CTC, and an unsupported medical itemized deduction. The applicable computation uses the $32,200 joint standard deduction, the $7,746.26 overtime deduction, current brackets, and a $6,600 CTC that eliminates $4,586.51 of tentative tax." -us,scenario_026,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The submitted $5,352 is consistent with stopping near tentative tax rather than subtracting the nonrefundable CTC actually used. After the overtime deduction, tentative tax is $4,586.51, and the available $6,600 CTC reduces the requested output to $0." -us,scenario_026,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed expiration of the governing individual provisions and used a $16,600 standard deduction, personal exemptions, and a $1,000-per-child CTC. The applicable 2026 rules provide a $32,200 standard deduction, a $7,746.26 overtime deduction, and a $6,600 CTC that eliminates tentative tax." -us,scenario_026,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's invocation of personal exemptions and its positive balance reflect use of the wrong 2026 tax regime and failure to apply the full usable CTC. The applicable deductions produce $4,586.51 of tentative tax, all of which is offset by the $6,600 CTC." -us,scenario_026,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The $15,870 estimate is incompatible with married filing jointly on $82,300.52 of AGI and implies that the model failed to settle on the specified joint filing status or apply the stated deductions and credits. Joint brackets, the $32,200 standard deduction, the overtime deduction, and the usable CTC produce no tax before refundable credits." -us,scenario_026,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a $30,000 standard deduction, omitted the $7,746.26 overtime deduction, and applied an obsolete $1,000-per-child CTC. The applicable $6,600 CTC fully offsets the $4,586.51 tentative tax produced after the correct deductions." -us,scenario_026,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The terse $4,520 answer is consistent with reporting tentative tax while failing to subtract the nonrefundable CTC actually used. The household has $6,600 of CTC available, which offsets all $4,586.51 of tentative tax." -us,scenario_026,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly used a post-sunset framework with personal exemptions, medical itemization, and a $1,000-per-child CTC. Under the applicable 2026 deductions and credit amount, taxable income is $42,354.26 and the $6,600 CTC eliminates all $4,586.51 of tentative tax." -us,scenario_026,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted the overtime premium by adding $7,746 to annual gross wages even though gross wages already include it, then used pre-TCJA exemptions, brackets, and a $1,000-per-child CTC instead of deducting qualifying overtime. The correct deductions yield $4,586.51 of tentative tax, fully offset by the $6,600 CTC." -us,scenario_026,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract." -us,scenario_026,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model incorrectly concluded that tax liability exceeded the entire $6,600 CTC. The nonrefundable credit does not absorb all $6,600; the unused $2,013.49 is refundable." -us,scenario_026,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model invented an approximate $1,000 refundable amount per child instead of calculating the unused CTC after the nonrefundable offset. That calculation leaves exactly $2,013.49 refundable, not $3,000." -us,scenario_026,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a $6,000 total CTC instead of the 2026 $6,600 amount and incorrectly treated the credit as fully absorbed by tax liability. Correctly applying the $2,200-per-child credit leaves $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used a $2,000-per-child CTC and concluded that tax liability absorbed it completely. The 2026 credit is $2,200 per child, and $2,013.49 remains after the nonrefundable portion offsets liability." -us,scenario_026,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an overstated tax-before-credit figure of about $5,450. The applicable liability absorbs only $4,586.51 of the $6,600 CTC, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a $6,000 CTC and approximate standard deduction and bracket values, producing an overstated $5,815 tax liability. The 2026 $6,600 CTC less the correctly computed $4,586.51 nonrefundable use yields a $2,013.49 refundable credit." -us,scenario_026,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $7,746 FLSA overtime premium on top of annual gross wages even though gross wages already include overtime, inflating AGI and tax liability. It then incorrectly treated the entire CTC as nonrefundable instead of refunding the remaining $2,013.49." -us,scenario_026,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model correctly eliminated EITC but incorrectly asserted that the nonrefundable CTC fully offsets the available child credit. After the liability offset, $2,013.49 of CTC remains refundable." -us,scenario_026,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a reversion to a $1,000-per-child, $3,000 total CTC. The applicable 2026 credit is $2,200 per child, and its refundable remainder is $2,013.49." -us,scenario_026,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The zero answer omits the refundable CTC generated by the three qualifying children. The CTC remaining after the nonrefundable liability offset is $2,013.49." -us,scenario_026,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $3,000 total Child Tax Credit rather than the applicable $6,600 credit. Although EITC is zero, $2,013.49 of the CTC remains refundable after offsetting tax liability." -us,scenario_026,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $61,093 without support from the listed deductions and consequently awarded a spurious $930 EITC. EITC is zero, while the correctly calculated refundable CTC alone equals $2,013.49." -us,scenario_026,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The zero answer omits the unused portion of the Child Tax Credit. Applying the nonrefundable CTC against liability leaves $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model overstated tax liability at $5,676.12 and also used a $6,000 rather than $6,600 total CTC. The correct nonrefundable use is $4,586.51, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated the children's being over age six as eliminating the refundable CTC. All three children are under 17 and qualify, producing a $2,013.49 refundable remainder after the liability offset." -us,scenario_026,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model treated income above the EITC range as eliminating every refundable credit. The household still receives $2,013.49 of refundable CTC from the three qualifying children." -us,scenario_026,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model applied a $1,000-per-child post-2025 credit and treated it as nonrefundable. The applicable 2026 CTC is $2,200 per child, with $2,013.49 refundable after the nonrefundable offset." -us,scenario_026,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model identified the correct $6,600 CTC and refundable pathway but understated the nonrefundable portion used against tax by $45.27. The resulting refundable remainder is $2,013.49, not $2,058.76." -us,scenario_026,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly treated this income level as disqualifying the household from refundable CTC. Income eliminates EITC but does not eliminate the $2,013.49 refundable CTC remainder." -us,scenario_026,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a $3,000 total CTC and therefore concluded that tax liability exhausted it. The applicable $6,600 CTC exceeds the nonrefundable amount used by $2,013.49." -us,scenario_026,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model both double-counted the overtime premium in earned income and used a $3,000 total CTC. Gross wages already include overtime, and the applicable $6,600 CTC leaves $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for federal_refundable_credits. It therefore failed the required output contract. -us,scenario_026,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly used the $6,600 CTC but overstated tax liability at $5,873. The correct nonrefundable use is $4,586.51, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model correctly excluded EITC but incorrectly set the additional CTC to zero. The unused CTC after offsetting liability is refundable and equals $2,013.49." -us,scenario_026,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a $6,000 total CTC and overstated the nonrefundable amount used at $5,479. The 2026 $6,600 credit less $4,586.51 used against liability produces a $2,013.49 refundable credit." -us,scenario_026,free_school_meals_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model reversed the household's position relative to the school-meal income thresholds, calling $85,209 well below the free-meal standard despite its ratio of 220% of the federal poverty guideline. With no SNAP/TANF categorical eligibility and no universal free-meals policy in North Carolina, the household pays full price." +us,scenario_025,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model correctly stated that the 61-year-old head does not meet Medicare eligibility, but then submitted value = 1, which denotes Yes. It inverted its own eligibility conclusion when encoding the numeric output." +us,scenario_025,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly added an Ohio employee payroll tax, even though Ohio has no mandatory employee-side state payroll tax included here, and invented an unexplained adjustment. It then submitted $9,432.76 despite its own displayed components totaling roughly $5,432.76; the federal Social Security and Medicare components alone total $4,798.48." +us,scenario_025,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model identified the correct taxable wage base and correctly calculated approximately $3,888.95 of Social Security tax plus $909.51 of Medicare tax. It then replaced their $4,798.46 sum with $4,800.14 on the unsupported claim of precise rounding, creating a final-answer arithmetic error." +us,scenario_025,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the head’s $21,208 employer-sponsored insurance premium from the spouse’s $62,725 wages. That premium belongs to a different person and is not stated to be a pre-tax payroll deduction, so the full $62,725 wage amount remains subject to Social Security and Medicare tax." +us,scenario_025,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct rule—7.65% employee payroll tax on the spouse’s $62,725 wages with no Additional Medicare Tax—but submitted $9,571.50 instead of performing that multiplication. The stated computation yields $4,798.46 before trace-level rounding, not the submitted amount." +us,scenario_025,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model said it computed payroll tax on annual wages for both spouses even though only the spouse has listed wages and every unlisted numeric input must be zero. It therefore introduced a nonexistent wage base for the head instead of applying Social Security and Medicare tax solely to the spouse’s $62,725." +us,scenario_025,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract rather than completing the calculation." +us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly identified the pension credit but replaced the 2026 Ohio schedule with a 2.75% calculation above an assumed $26,050 zero bracket, then introduced an unexplained downward adjustment from about $1,576 to $875. The actual bracket calculation on $91,125.30 is $2,121.57 before the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly started Ohio taxable income from federal taxable income after a federal standard deduction; Ohio starts here from $94,925.30 of Ohio AGI and subtracts $3,800 of Ohio personal exemptions. It also used obsolete rates reaching 5.75% and omitted the $200 retirement-income credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used the wrong $3,700 exemption total and explicitly subtracted no nonrefundable credits. The household receives two $1,900 exemptions and the $200 retirement-income credit, while the applicable schedule produces $2,121.57 before that credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model identified the correct $3,800 exemptions and $200 retirement credit but its stated 2.75% shortcut yields $1,590 after the credit, not its submitted $2,426. It failed to apply the actual Ohio schedule, which yields $2,121.57 before the credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used approximately $5,000 of personal exemptions instead of the correct $3,800 and did not separately apply the $200 retirement-income credit. It also substituted a rough 2.75% calculation for the Ohio schedule." +us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used two $2,400 personal exemptions instead of two $1,900 exemptions, understating taxable income by $1,000. It then applied a simple 2.75% rate above $26,050 rather than the schedule that produces $2,121.57 before the correctly identified $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model invented a $250 retirement credit and an additional senior credit even though neither spouse qualifies for an Ohio senior credit at ages 61 and 57. The applicable nonrefundable reduction is the $200 retirement-income credit, and the personal exemptions total $3,800." +us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used only one $2,400 personal exemption, applied obsolete graduated rates of 2.75%, 3.226%, and 3.688%, and omitted the retirement-income credit. The correct inputs are two $1,900 exemptions followed by the 2026 schedule and a $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model deducted the separately reported $21,208 employer-sponsored insurance premium from wages even though gross wages are the annual taxable input and no such Ohio AGI subtraction applies. It also created a medical-expense subtraction and used $4,800 of exemptions rather than $3,800, driving taxable income far below $91,125.30." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $1,695 estimate does not follow the traced Ohio computation. On $91,125.30 of taxable income, the schedule yields $2,121.57 before the $200 retirement credit, not a rough rate-based estimate." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model computed only a rough 2.75% tax above $26,050 and stopped at $1,762. It omitted the $200 retirement-income credit from its reasoning and did not use the schedule that produces $2,121.57 before credits." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted two $2,700 exemptions instead of two $1,900 exemptions, understating taxable income by $1,600. It also used a flat 2.75% shortcut and omitted the $200 retirement-income credit." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly derived $91,125 of taxable income and recognized the $200 retirement credit, but it applied 2.75% only to income above $26,050. The actual 2026 Ohio schedule yields $2,121.57 before the credit, producing $1,921.57 after it." +us,scenario_025,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted an estimated federal standard deduction from Ohio AGI and also used $4,800 rather than $3,800 of Ohio exemptions. Ohio taxable income is $91,125.30, not $60,125, and the $200 retirement credit then applies after computing bracket tax." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model invented a full offset based on a nonexistent low-income or family credit. The household has $2,121.57 of Ohio tax before credits and only the $200 retirement-income credit reduces it, leaving $1,921.57." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked an Ohio standard deduction and unspecified assumed credits instead of deriving the liability from Ohio AGI, two $1,900 exemptions, the tax schedule, and the $200 retirement credit. Its $1,340 estimate therefore omits the controlling computation steps." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model invented a $230.625 medical deduction, reducing taxable income below the traced $91,125.30. No such subtraction enters this calculation; applying the schedule to $91,125.30 gives $2,121.57 before the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model used two $2,400 exemptions instead of two $1,900 exemptions and explicitly omitted the available $200 retirement-income credit. It also replaced the applicable Ohio schedule with a 2.75% calculation above $26,050." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly used $3,800 of exemptions and the $200 retirement credit but computed pre-credit tax as $2,150.25 instead of $2,121.57. Its projected schedule therefore overstated Ohio bracket tax by $28.68." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the entire $32,200 private pension from Ohio taxable income even though it is included in the $94,925.30 Ohio AGI. Ohio provides a $200 retirement-income credit rather than a pension exclusion here, and the exemptions are $1,900 each rather than $2,150." +us,scenario_025,state_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model invented deductions or credits sufficient to eliminate all Ohio taxable income or liability. The household instead has $91,125.30 of taxable income and only a $200 retirement-income credit against $2,121.57 of pre-credit tax." +us,scenario_025,state_income_tax_before_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model used $4,800 of exemptions rather than $3,800 and asserted that no nonrefundable credit applies. The $32,200 pension qualifies the household for a $200 retirement-income credit, and the applicable schedule is not the model's simple 2.75% calculation." +us,scenario_025,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used two $2,650 exemptions and an obsolete two-rate structure with 3.5% beginning after $46,100. It also omitted the $200 retirement-income credit; the correct exemption total is $3,800 and pre-credit tax is $2,121.57." +us,scenario_025,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model used approximately $4,800 of exemptions instead of $3,800 and stated that no nonrefundable credits apply. The household qualifies for the $200 retirement-income credit, and Ohio's schedule produces $2,121.57 before that credit." +us,scenario_025,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model used $5,000 of personal exemptions instead of the correct $3,800, understating taxable income by $1,200. It also applied a simple 2.75% rate above $26,050 and omitted the $200 retirement-income credit." +us,scenario_025,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly derived the $91,125 taxable-income base and the $200 retirement credit but applied 2.75% only to the amount above $26,050. The applicable schedule produces $2,121.57 before the credit, not $1,789.56." +us,scenario_025,state_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model invented senior or aged adjustments that reduce liability to zero even though ages 61 and 57 do not supply such an offset. The household receives two $1,900 personal exemptions and a $200 retirement-income credit, leaving $1,921.57." +us,scenario_025,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used an unspecified estimated bracket calculation that overstated the liability and did not account explicitly for the $3,800 personal exemptions or $200 retirement-income credit. Those steps yield $91,125.30 of taxable income, $2,121.57 before credits, and $1,921.57 after the credit." +us,scenario_025,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented a $9,300 Ohio married-filing-jointly standard deduction, used a rate structure that taxes the nominal zero-bracket income, and applied a senior credit to a 61-year-old. Ohio instead subtracts $3,800 of personal exemptions, applies the schedule to $91,125.30, and subtracts the $200 retirement-income credit." +us,scenario_026,child1_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model applied only North Carolina's CHIP income ceiling and concluded that income below 216% of FPL established eligibility. It omitted the prior Medicaid screen: Child 1 qualifies under the OLDER_CHILD Medicaid category, which makes the child ineligible for CHIP." +us,scenario_026,child1_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model treated income below 211% of FPL as sufficient for CHIP eligibility and also used an incorrect household-income figure. It failed to apply the controlling Medicaid exclusion: Child 1 is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP. +us,scenario_026,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_chip_eligible. The required output is 0 because Child 1 qualifies for Medicaid under the OLDER_CHILD category, and Medicaid eligibility excludes CHIP eligibility." +us,scenario_026,child1_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model applied a 138% FPL limit to an 11-year-old, substituting the adult-expansion threshold for North Carolina's older-child Medicaid threshold. At the traced MAGI of 2.13 times FPL, Child 1 satisfies the applicable older-child income limit." +us,scenario_026,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an incorrect 200% FPL ceiling and an incorrect family-of-five poverty guideline, then treated employer-sponsored insurance as additional evidence against Medicaid eligibility. North Carolina's older-child limit admits Child 1 at 2.13 times FPL, and existing employer coverage does not negate Medicaid eligibility." +us,scenario_026,child1_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,The model applied a 138% FPL adult-expansion limit instead of North Carolina's Medicaid income limit for children ages 6 through 18. Child 1's traced MAGI of 2.13 times FPL is within the older-child threshold. +us,scenario_026,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,The model substituted the 138% FPL adult Medicaid threshold for the older-child category applicable to this 11-year-old. The applicable North Carolina child threshold covers the traced 2.13-times-FPL MAGI level. +us,scenario_026,child1_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model overstated the household's relevant MAGI as roughly 260% FPL. The engine's MAGI calculation is 2.13 times FPL, which falls within North Carolina's older-child Medicaid limit." +us,scenario_026,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,The model invented an uninsured-status requirement for Medicaid and treated employer-sponsored insurance as disqualifying. Child 1's existing ESI does not block Medicaid eligibility when the older-child categorical and income tests are satisfied. +us,scenario_026,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model added the listed FLSA overtime premium to gross wages even though gross wages are annual totals including overtime, thereby double-counting income. It also failed to use the traced MAGI-to-FPL result of 2.13, which satisfies the older-child Medicaid limit." +us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a 133% FPL threshold to Child 1, confusing the applicable older-child limit with a lower Medicaid threshold. North Carolina covers this older child at the traced MAGI of 2.13 times FPL." +us,scenario_026,child1_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model used an incorrect 147% FPL ceiling for North Carolina's older-child Medicaid category. The applicable limit covers Child 1 at the engine-derived MAGI level of 2.13 times FPL. +us,scenario_026,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies a child Medicaid income ceiling below 2.13 times FPL. North Carolina's older-child category permits eligibility at that traced MAGI level, so the income-threshold conclusion was reversed." +us,scenario_026,child1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model applied a 133% FPL Medicaid limit to an 11-year-old and routed the child to CHIP instead. The applicable North Carolina older-child Medicaid threshold covers the traced 2.13-times-FPL MAGI. +us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used a 133% FPL ceiling rather than the North Carolina older-child Medicaid limit. Child 1 meets the applicable income test at 2.13 times FPL. +us,scenario_026,child1_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,The answer implies that North Carolina's older-child Medicaid limit is below 2.13 times FPL. The applicable threshold covers this 11-year-old at exactly that engine-calculated MAGI level. +us,scenario_026,child1_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,"The model treated employer-sponsored insurance as a Medicaid disqualifier. Medicaid eligibility under the traced older-child pathway depends on the categorical and MAGI tests, both of which Child 1 satisfies; ESI does not defeat eligibility." +us,scenario_026,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,The model assumed the school-age Medicaid threshold was below the household's income level without applying North Carolina's older-child limit. The traced MAGI is 2.13 times FPL and remains within that category's threshold. +us,scenario_026,child1_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model relied on a generic income impression and the absence of special status instead of applying the ordinary older-child MAGI pathway. As an 11-year-old dependent at 2.13 times FPL, Child 1 meets North Carolina's applicable child Medicaid test." +us,scenario_026,child1_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,The model placed North Carolina's school-age Medicaid/CHIP income limit below the household's traced MAGI. The Medicaid older-child threshold covers Child 1 at 2.13 times FPL. +us,scenario_026,child1_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model underestimated North Carolina's Medicaid income threshold for an 11-year-old. The older-child category admits the engine-calculated MAGI of 2.13 times FPL. +us,scenario_026,child1_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,The model incorrectly concluded that 2.13-times-FPL MAGI exceeds North Carolina's limit for an 11-year-old. That income level falls within the applicable older-child Medicaid threshold. +us,scenario_026,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model placed the North Carolina child Medicaid limit below the traced 2.13-times-FPL income level. The older-child threshold applicable at age 11 covers that MAGI. +us,scenario_026,child1_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model both underestimated North Carolina's older-child Medicaid income limit and treated ESI as adverse to eligibility. Child 1 qualifies at 2.13 times FPL, and employer-sponsored coverage does not disqualify the child from Medicaid." +us,scenario_026,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,The model applied a 133% FPL limit to the Medicaid category for children ages 6 through 18. North Carolina's applicable older-child limit covers Child 1 at 2.13 times FPL. +us,scenario_026,child1_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an inapplicable 133% FPL base threshold and overstated the income ratio as roughly 240% FPL. The engine derives MAGI at 2.13 times FPL, within North Carolina's older-child limit." +us,scenario_026,child1_medicaid_eligible,inkling,llm_error,thresholds_rates,False,The model imposed a 133% FPL ceiling on Medicaid for ages 6 through 18 and therefore misrouted the coverage analysis toward CHIP. North Carolina's older-child Medicaid threshold covers the traced MAGI of 2.13 times FPL. +us,scenario_026,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child1_medicaid_eligible, violating the required output contract. The required substantive output was 1 because Child 1 meets the older-child Medicaid test at 2.13 times FPL." +us,scenario_026,child1_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,The model used a 133%/138% FPL ceiling for an 11-year-old and also overstated MAGI as about 226% FPL. The traced MAGI is 2.13 times FPL and falls within North Carolina's older-child Medicaid limit. +us,scenario_026,child1_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model assumed the household was above North Carolina's Medicaid-for-children limit without applying the older-child threshold. At the traced MAGI of 2.13 times FPL, Child 1 satisfies that limit." +us,scenario_026,child1_medicaid_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model correctly reasoned that the stated child income range includes the household, then reversed the result solely because Child 1 has employer-sponsored insurance. Existing ESI does not disqualify a child who satisfies North Carolina's older-child Medicaid categorical and MAGI tests." +us,scenario_026,child1_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,The model implicitly placed North Carolina's older-child Medicaid ceiling below 2.13 times FPL. The applicable threshold covers Child 1 at that traced MAGI level. +us,scenario_026,child2_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated household MAGI below the CHIP income limit as sufficient for CHIP eligibility and omitted the prerequisite that the child not qualify for Medicaid. Child 2 qualifies for Medicaid under the OLDER_CHILD category, which makes the CHIP result 0." +us,scenario_026,child2_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model applied the NC Health Choice income threshold without first testing Medicaid eligibility. Child 2 is Medicaid-eligible under the OLDER_CHILD category and therefore excluded from CHIP, irrespective of its separate $61,093 income calculation." +us,scenario_026,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable output for child2_chip_eligible. The required derivation is Medicaid eligibility under the OLDER_CHILD category followed by CHIP exclusion, yielding 0." +us,scenario_026,child2_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,The model treated its approximate 211% FPL figure as an exclusionary cutoff and calculated the household at 219% FPL. The applicable OLDER_CHILD test places Medicaid MAGI at 2.13 times FPL and within North Carolina's limit. +us,scenario_026,child2_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model incorrectly used employer-sponsored insurance as an additional reason to deny Medicaid and also invented a second wage earner despite the head having no listed wages. Existing employer-sponsored coverage does not bar eligibility, and the household passes the OLDER_CHILD income test at 2.13 times FPL." +us,scenario_026,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model alternated among 215%, 211%, and 133% FPL and ultimately applied the 133% adult-expansion-style threshold to an 11-year-old. Child 2 belongs to the OLDER_CHILD category, whose applicable North Carolina limit admits MAGI of 2.13 times FPL." +us,scenario_026,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied a 138% FPL Medicaid cutoff to an 11-year-old and treated the higher child limit as CHIP-only. PolicyEngine applies North Carolina's OLDER_CHILD Medicaid threshold, under which 2.13 times FPL qualifies." +us,scenario_026,child2_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model overstated the household income level as roughly 260% FPL. The Medicaid MAGI calculation produces 2.13 times FPL, and that amount passes the OLDER_CHILD threshold." +us,scenario_026,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model imposed an uninsured or uncovered requirement on Medicaid. Employer-sponsored insurance does not disqualify Child 2 from Medicaid, and the child passes the OLDER_CHILD income test." +us,scenario_026,child2_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model counted the listed overtime premium on top of gross wages even though gross wages already include all overtime pay, and it understated the family-of-five poverty guideline. The resulting Medicaid MAGI is 2.13 times FPL and falls within the OLDER_CHILD limit." +us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a 133% FPL cutoff to Child 2. An 11-year-old dependent is tested under North Carolina's OLDER_CHILD category, whose income limit admits the household's 2.13-times-FPL MAGI." +us,scenario_026,child2_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a 133% FPL threshold for an 11-year-old. The applicable category is OLDER_CHILD, and its North Carolina Medicaid limit permits eligibility at 2.13 times FPL." +us,scenario_026,child2_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies an income cutoff below the household's 2.13-times-FPL Medicaid MAGI. North Carolina's OLDER_CHILD limit includes that income level, so the income test is satisfied." +us,scenario_026,child2_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied the 133% FPL threshold and diverted the child to CHIP. Child 2 is in the OLDER_CHILD Medicaid category, whose applicable limit includes MAGI at 2.13 times FPL." +us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model tested the child against a 133% FPL limit. North Carolina's OLDER_CHILD Medicaid threshold, not that lower threshold, applies to this 11-year-old and admits 2.13 times FPL." +us,scenario_026,child2_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,The answer implies that North Carolina's child Medicaid limit is below 2.13 times FPL. The applicable OLDER_CHILD limit includes that MAGI level. +us,scenario_026,child2_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,The model applied a 133% FPL Medicaid cutoff to ages 6–18 and treated the higher child threshold as CHIP-only. The OLDER_CHILD Medicaid category applies and permits eligibility at the household's 2.13-times-FPL MAGI. +us,scenario_026,child2_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,The answer assumed the Medicaid threshold was below the household's income without applying the OLDER_CHILD limit. That category admits the calculated Medicaid MAGI of 2.13 times FPL. +us,scenario_026,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model treated the absence of a special-status fact as defeating eligibility and failed to apply the ordinary dependent-child pathway. Child 2 qualifies through the OLDER_CHILD category because the household's 2.13-times-FPL MAGI is within its limit. +us,scenario_026,child2_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,The model concluded that earnings exceeded the child limit without applying North Carolina's OLDER_CHILD threshold. The Medicaid MAGI is 2.13 times FPL and is within that threshold. +us,scenario_026,child2_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model estimated an income limit below the household's Medicaid MAGI. The applicable OLDER_CHILD limit includes MAGI at 2.13 times FPL. +us,scenario_026,child2_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model misapplied North Carolina's income limit for an 11-year-old. Under the OLDER_CHILD category, the household's 2.13-times-FPL MAGI satisfies the limit." +us,scenario_026,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The answer implies a child Medicaid cutoff below 2.13 times FPL. North Carolina's applicable OLDER_CHILD threshold admits that income level. +us,scenario_026,child2_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"The model treated both income and existing employer-sponsored insurance as disqualifying. Employer-sponsored coverage is not a Medicaid bar, and 2.13 times FPL passes the OLDER_CHILD income limit." +us,scenario_026,child2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,The model applied a 133% FPL cutoff to a child in the OLDER_CHILD category. The applicable North Carolina threshold permits eligibility at 2.13 times FPL. +us,scenario_026,child2_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated the household's income level as roughly 240% FPL and applied a 133% base cutoff. The Medicaid MAGI is 2.13 times FPL, which satisfies the OLDER_CHILD threshold." +us,scenario_026,child2_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model applied the 133% FPL threshold to an 11-year-old and relied on employer-sponsored insurance in discussing coverage eligibility. The OLDER_CHILD Medicaid threshold applies, admits 2.13 times FPL, and does not require the child to be uninsured." +us,scenario_026,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for child2_medicaid_eligible. It therefore failed the required output contract rather than performing the OLDER_CHILD eligibility calculation. +us,scenario_026,child2_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model used a 133% or 138% FPL cutoff for an 11-year-old and overstated Medicaid MAGI as roughly 226% FPL. The correct category is OLDER_CHILD, and the calculated 2.13-times-FPL MAGI passes its limit." +us,scenario_026,child2_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,The model asserted that income was above the Medicaid for Children limit without applying the OLDER_CHILD threshold. That threshold includes the household's 2.13-times-FPL Medicaid MAGI. +us,scenario_026,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit to Child 2 and used an understated family-of-five poverty guideline. An 11-year-old is evaluated under the OLDER_CHILD category, whose North Carolina limit admits Medicaid MAGI at 2.13 times FPL." +us,scenario_026,child2_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,The answer implies an income threshold below the household's 2.13-times-FPL Medicaid MAGI. The applicable North Carolina OLDER_CHILD limit includes that income level. +us,scenario_026,child3_chip_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model treated an asserted CHIP income band as sufficient for eligibility and never applied the rule excluding children who qualify for Medicaid. Child 3 is Medicaid-eligible under the OLDER_CHILD category and therefore is not CHIP-eligible. +us,scenario_026,child3_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model concluded that income below its asserted CHIP limit established eligibility, omitting the prerequisite that the child not qualify for Medicaid. Child 3's OLDER_CHILD Medicaid eligibility automatically precludes CHIP." +us,scenario_026,child3_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model used household income below the CHIP threshold as the complete eligibility test. It failed to apply the Medicaid-exclusion step: Child 3 qualifies for Medicaid under the OLDER_CHILD category and thus cannot qualify for CHIP. +us,scenario_026,child3_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model checked age, Medicare status, and a CHIP income range but omitted the controlling Medicaid-status test. Child 3 is Medicaid-eligible under the OLDER_CHILD category, which makes the child ineligible for CHIP even when the asserted CHIP income test is satisfied." +us,scenario_026,child3_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,The model applied a roughly 138% FPL Medicaid cutoff to a 9-year-old and routed higher-income children to CHIP. North Carolina's applicable 2026 OLDER_CHILD Medicaid limit includes this dependent child at 2.13 times FPL. +us,scenario_026,child3_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,The model imposed a 207% FPL cutoff and therefore excluded the child based on its own MAGI estimate. The applicable OLDER_CHILD limit includes the engine-calculated 2.13-times-FPL income level; employer-sponsored insurance does not negate eligibility. +us,scenario_026,child3_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,The model used an incorrect approximately 211% FPL limit and compared it with gross wages rather than the engine's MAGI-to-FPL result. Child 3 qualifies in the OLDER_CHILD category at 2.13 times FPL. +us,scenario_026,child3_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,The model applied the obsolete 133%-plus-disregard framework to the age-6-to-18 category and incorrectly treated the remaining coverage range as CHIP. The applicable 2026 OLDER_CHILD Medicaid threshold covers 2.13 times FPL. +us,scenario_026,child3_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model overstated the household's income level as about 260% FPL and then declared it above the child limit. The engine calculates MAGI at 2.13 times FPL, which satisfies North Carolina's OLDER_CHILD threshold." +us,scenario_026,child3_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model incorrectly treated employer-sponsored insurance as a bar to Medicaid eligibility and also used an income cutoff below the applicable OLDER_CHILD limit. Existing ESI does not prevent this dependent child from qualifying at 2.13 times FPL. +us,scenario_026,child3_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model applied a 138% FPL limit to a 9-year-old and improperly added the separately listed overtime premium to annual wages even though gross wages already include overtime. Under the applicable OLDER_CHILD rule, the engine's 2.13-times-FPL MAGI qualifies." +us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,The model used 133% FPL as the controlling North Carolina limit for this older child. The applicable 2026 OLDER_CHILD threshold covers the engine-calculated MAGI of 2.13 times FPL. +us,scenario_026,child3_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a 133% FPL cutoff and an income/FPL calculation inconsistent with the trace. Child 3's controlling MAGI level is 2.13 times FPL, which falls within the applicable OLDER_CHILD limit." +us,scenario_026,child3_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,The answer assumes the household exceeds the governing child Medicaid cutoff. The applicable North Carolina OLDER_CHILD threshold includes Child 3 at the engine-calculated 2.13 times FPL. +us,scenario_026,child3_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model applied a 133% FPL Medicaid limit and routed the child to CHIP. North Carolina's applicable 2026 OLDER_CHILD Medicaid limit covers this 9-year-old at 2.13 times FPL. +us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a 133% FPL cutoff and an income calculation inconsistent with the traced MAGI ratio. The controlling result is 2.13 times FPL, within the OLDER_CHILD Medicaid limit." +us,scenario_026,child3_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,The model's conclusion assumes an income limit below 2.13 times FPL. The applicable 2026 North Carolina OLDER_CHILD limit includes Child 3 at that MAGI level. +us,scenario_026,child3_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model incorrectly used 133% FPL for Medicaid and 210% FPL as a separate CHIP ceiling, then treated ESI as adverse. The applicable OLDER_CHILD Medicaid rule includes this dependent 9-year-old at 2.13 times FPL, and ESI is not an eligibility bar." +us,scenario_026,child3_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,The model assumed earnings exceeded the benchmark Medicaid threshold without applying North Carolina's applicable OLDER_CHILD limit. The engine-calculated MAGI of 2.13 times FPL satisfies that limit. +us,scenario_026,child3_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,The model inferred ineligibility from wages and the absence of a special status instead of testing the ordinary OLDER_CHILD pathway. Child 3 qualifies through that pathway at 2.13 times FPL without SSI or another special status. +us,scenario_026,child3_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model treated annual earnings as exceeding North Carolina's child Medicaid limit. The relevant comparison is the traced MAGI of 2.13 times FPL against the 2026 OLDER_CHILD limit, which Child 3 satisfies." +us,scenario_026,child3_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model estimated a child Medicaid threshold below the household's income instead of applying the governing OLDER_CHILD threshold. That threshold includes Child 3 at 2.13 times FPL. +us,scenario_026,child3_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,The model incorrectly concluded that MAGI exceeds the limit for a 9-year-old. The engine's 2.13-times-FPL MAGI falls within North Carolina's applicable OLDER_CHILD Medicaid limit. +us,scenario_026,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model assumed the household was above North Carolina's child Medicaid limit. Applying the 2026 OLDER_CHILD threshold to the traced 2.13-times-FPL MAGI makes Child 3 eligible. +us,scenario_026,child3_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model used an income cutoff below the applicable OLDER_CHILD limit and treated ESI as supporting ineligibility. Child 3 qualifies at 2.13 times FPL, and employer-sponsored coverage does not bar Medicaid eligibility." +us,scenario_026,child3_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,The model applied a 133% FPL limit to North Carolina's age-6-to-18 Medicaid category. The applicable 2026 OLDER_CHILD limit covers the traced MAGI of 2.13 times FPL. +us,scenario_026,child3_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model both overstated the income ratio as about 240% FPL and used a roughly 133% FPL cutoff. The engine calculates 2.13 times FPL, which is within the applicable OLDER_CHILD Medicaid limit." +us,scenario_026,child3_medicaid_eligible,inkling,llm_error,categorical_eligibility,False,"The model used a 133% FPL cutoff and treated employer-sponsored insurance as adverse to eligibility. North Carolina's applicable OLDER_CHILD limit includes the child at 2.13 times FPL, and ESI does not disqualify the child." +us,scenario_026,child3_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for child3_medicaid_eligible. It therefore failed the required output contract rather than completing the OLDER_CHILD eligibility determination. +us,scenario_026,child3_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model applied a 133% or 138% FPL cutoff and overstated household MAGI as about 226% FPL. The trace places MAGI at 2.13 times FPL, within the applicable OLDER_CHILD Medicaid limit." +us,scenario_026,child3_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,The model assumed household income was above the Medicaid-for-children limit without applying the governing age-specific threshold. The applicable OLDER_CHILD limit includes Child 3 at 2.13 times FPL. +us,scenario_026,child3_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used an incorrect 133% FPL threshold and an incorrect poverty guideline for a five-person household, then treated ESI as supporting exclusion. The applicable OLDER_CHILD Medicaid limit covers the traced 2.13-times-FPL MAGI, and ESI does not bar eligibility." +us,scenario_026,child3_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model assumed household income exceeded the child Medicaid limit. Under North Carolina's applicable 2026 OLDER_CHILD category, the engine-calculated MAGI of 2.13 times FPL qualifies." +us,scenario_026,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model omitted the $7,746.26 overtime deduction and inconsistently restricted the nonrefundable child tax credit to roughly $1,500. The deduction produces $4,586.51 of tentative tax, all of which is offset by the available $6,600 child tax credit." +us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly concluded in its own reasoning that the child tax credit absorbs all tentative tax, then submitted $4,107 instead of zero. It also omitted the overtime deduction, but even its overstated tentative liability was fully covered by the child tax credit." +us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model omitted the overtime deduction and treated only about $605 of the child tax credit as nonrefundable by incorrectly reserving refundable portions first. Nonrefundable CTC usage equals the $4,586.51 tentative liability, reducing this output to zero." +us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model omitted the overtime deduction and incorrectly treated the $1,700-per-child refundability cap as limiting how much CTC can offset tentative tax. The available $6,600 credit first offsets the full $4,586.51 liability, leaving zero before refundable credits." +us,scenario_026,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $7,746 overtime premium by adding it to gross wages even though the stated annual wages already include overtime, and it failed to apply that amount as the overtime deduction. It then contradicted its own $446 post-CTC calculation by submitting $4,200; the correct deduction and CTC application reduce the output to zero." +us,scenario_026,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a pre-TCJA regime with personal exemptions, a $1,000-per-child CTC, and obsolete brackets instead of the applicable 2026 parameters. It also omitted the overtime deduction; under the applicable rules, tentative tax is $4,586.51 and the $6,600 CTC eliminates it." +us,scenario_026,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The submitted $5,352 leaves essentially the tentative income tax unpaid and therefore fails to apply the nonrefundable child tax credit correctly. After the overtime deduction, tentative tax is $4,586.51, and the available $6,600 CTC offsets it in full." +us,scenario_026,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed expiration-based pre-TCJA parameters, including personal exemptions and a $1,000-per-child CTC. It also omitted the overtime deduction; the applicable 2026 computation yields $4,586.51 of tentative tax fully offset by the $6,600 CTC." +us,scenario_026,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The answer invokes personal exemptions and unspecified standard or itemized deductions, reflecting the wrong 2026 parameter regime, and does not account for the overtime deduction. The applicable computation produces $4,586.51 of tentative tax, which the $6,600 child tax credit eliminates." +us,scenario_026,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The $15,870 result is incompatible with married-joint tax treatment of $82,300.52 AGI and implies that the model did not actually apply the stated filing status, deductions, brackets, and credits. The joint standard deduction plus overtime deduction yields $4,586.51 of tentative tax, fully offset by the child tax credit." +us,scenario_026,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an understated joint standard deduction, omitted the overtime deduction, and applied an obsolete $1,000-per-child CTC instead of $2,200 per child. The correct tentative tax is $4,586.51 and the $6,600 CTC eliminates it." +us,scenario_026,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The terse calculation leaves approximately the tentative federal tax outstanding, showing that it failed to apply the child tax credit against liability and omitted the overtime deduction. The deduction produces $4,586.51 of tentative tax, all offset by the available $6,600 CTC." +us,scenario_026,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied sunset-era personal exemptions and a $1,000-per-child CTC, and it omitted the $7,746.26 overtime deduction. Under the applicable 2026 rules, tentative tax is $4,586.51 and the $6,600 CTC offsets the entire amount." +us,scenario_026,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted the overtime premium by adding it to annual wages even though annual wages already include overtime, then failed to deduct the overtime amount. It also used obsolete personal exemptions and a $1,000-per-child CTC; the applicable calculation leaves $4,586.51 before credits and zero after the nonrefundable CTC." +us,scenario_026,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not be parsed or evaluated as a substantive estimate." +us,scenario_026,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model used $3,700 as the entire married-joint standard deduction, omitted the traditional 401(k), IRA, and overtime deductions, and used $3,700 as the child tax credit. The correct deductions produce $4,586.51 of tentative tax, which the actual $6,600 CTC eliminates." +us,scenario_026,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model incorrectly concluded that pre-credit tax absorbed the entire $6,600 CTC. Only $4,586.51 is used nonrefundably, leaving $2,013.49 as refundable CTC." +us,scenario_026,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model assigned an unsupported approximate refund of $1,000 per child instead of applying the CTC ordering and limitation rules. The $6,600 total CTC first offsets $4,586.51 of tax, leaving exactly $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used a $2,000-per-child CTC rather than the 2026 $2,200 amount and overstated the tax available to absorb it. The correct $6,600 credit leaves $2,013.49 refundable after its nonrefundable portion is applied." +us,scenario_026,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model used a $6,000 total CTC and incorrectly treated all of it as absorbed by regular tax. Under the 2026 $6,600 CTC, $2,013.49 remains refundable." +us,scenario_026,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model computed pre-credit federal tax as approximately $5,450 instead of $4,586.51, thereby understating the unused CTC. Subtracting the correct tax absorbed from the $6,600 CTC yields a refundable amount of $2,013.49." +us,scenario_026,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an approximate $30,000 standard deduction, approximate brackets, and a $2,000-per-child credit, producing excessive pre-credit tax and only $185 of unused CTC. Applying the 2026 parameters yields $4,586.51 used nonrefundably and $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model added the separately listed FLSA overtime premium to gross wages even though the prompt states that gross wages already include overtime, inflating AGI and tax liability. It also used a $6,000 CTC instead of $6,600, eliminating the $2,013.49 refundable remainder." +us,scenario_026,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model incorrectly asserted that the nonrefundable CTC fully offsets the available credit. The CTC exceeds the tax it can offset, and the unused $2,013.49 is refundable." +us,scenario_026,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly reverted to a $1,000-per-child 2017-law CTC and calculated a $3,000 total credit. The applicable 2026 credit is $2,200 per child, and the resulting $6,600 credit leaves $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The zero answer omits the refundable CTC pathway for the three qualifying children. The household has $2,013.49 of CTC remaining after the nonrefundable portion offsets federal tax." +us,scenario_026,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $3,000 total Child Tax Credit and therefore concluded that tax absorbed it. The 2026 total is $6,600, of which $2,013.49 remains refundable." +us,scenario_026,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $61,093 without support from the listed deductions and consequently awarded an erroneous $930 EITC. EITC is zero, while the correctly computed refundable CTC alone is $2,013.49." +us,scenario_026,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The zero answer omits the refundable portion of the Child Tax Credit. After the nonrefundable CTC offsets $4,586.51 of tax, $2,013.49 remains refundable." +us,scenario_026,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a $6,000 total CTC and overstated the tax absorbed as $5,676.12. The 2026 CTC totals $6,600, and only $4,586.51 is used nonrefundably, leaving $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated the children's being older than six as eliminating the refundable CTC. All three children are under 17 and qualify, producing $2,013.49 of refundable CTC after the nonrefundable credit is used." +us,scenario_026,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model treated household income as disqualifying the family from every refundable credit without applying the CTC rules. Although EITC is zero, the unused portion of the CTC is refundable and equals $2,013.49." +us,scenario_026,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model applied an obsolete post-2025 assumption of a $1,000 nonrefundable credit per child. The applicable 2026 CTC is $2,200 per child and includes a refundable portion, yielding $2,013.49." +us,scenario_026,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model identified the correct $6,600 CTC and refundable pathway but miscomputed the amount of credit left after tax. The correct unused and refundable balance is $2,013.49, not $2,058.76." +us,scenario_026,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly treated this income level as precluding refundable CTC eligibility. Income eliminates EITC but does not eliminate the $2,013.49 unused refundable portion of the CTC." +us,scenario_026,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a $3,000 total CTC and concluded that tax exceeded it. The applicable 2026 credit totals $6,600, leaving $2,013.49 refundable after offsetting tax." +us,scenario_026,federal_refundable_credits,grok-build-0.1,llm_error,period_annualization,False,"The model double-counted the $7,746 overtime premium even though annual gross wages already include overtime, producing erroneous earned income of $90,047. It also used a $3,000 CTC instead of the 2026 $6,600 total, thereby missing the $2,013.49 refundable balance." +us,scenario_026,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, so it failed the required output contract." +us,scenario_026,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the correct $6,600 total CTC but overstated the tax absorbed by the nonrefundable portion as $5,873. The correct absorbed amount is $4,586.51, leaving $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model correctly eliminated EITC based on income but incorrectly set the additional refundable CTC to zero. The unused CTC after offsetting tax is $2,013.49 and is refundable." +us,scenario_026,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a $6,000 CTC and overstated the nonrefundable amount used as $5,479. The 2026 CTC totals $6,600, with $4,586.51 used against tax and $2,013.49 refunded." +us,scenario_026,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model assigned $4,300 of refundable CTC without first applying the credit against federal tax liability. Of the $6,600 total CTC, $4,586.51 is used nonrefundably, so only $2,013.49 is refundable." +us,scenario_026,free_school_meals_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model reversed the income comparison by calling $85,208.82 well below the free-meal standard. That income is 220% of the family-of-five poverty guideline, and without SNAP/TANF categorical eligibility or universal free meals in North Carolina, the household must pay full price." us,scenario_026,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_026,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_026,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly computed federal FICA on the spouse's $85,209 of wages, then incorrectly added a fabricated North Carolina employee payroll tax component. The payroll_tax output excludes state income tax and North Carolina has no mandatory employee payroll tax component in this calculation." -us,scenario_026,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model reduced the FICA wage base by employer-sponsored health insurance premiums and also treated the FLSA overtime premium as part of wages before that subtraction. PolicyEngine applies employee Social Security and Medicare tax to the spouse's listed gross wages of $85,209, with no reduction for health insurance premiums and no separate addition of the overtime premium." -us,scenario_026,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model added the $7,746 FLSA overtime premium to the spouse's $85,209 gross wages and applied FICA to $92,955. The prompt states gross wages and salaries are annual totals including overtime pay, so the overtime premium is not a second payroll tax wage amount." -us,scenario_026,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model subtracted the head's employer-sponsored insurance premiums from the spouse's wages before applying FICA. Employee Social Security and Medicare tax are computed on the spouse's $85,209 wage base here, not on wages net of another household member's employer-sponsored insurance premiums." -us,scenario_026,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model identified the right tax base and components but made a numerical calculation error. Applying 6.2% Social Security and 1.45% Medicare to $85,209 yields about $6,518.47 in PolicyEngine's rounded computation, not $6,519.97." -us,scenario_026,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The submitted $10,618 is inconsistent with applying only employee FICA to the spouse's $85,209 of wages. The correct computation taxes only the spouse's wage earnings for employee Social Security and Medicare; it does not add employer payroll tax, income tax, state tax, or any non-wage household item." -us,scenario_026,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model applied FICA to $92,955 by adding the $7,746 FLSA overtime premium to the spouse's $85,209 gross wages. Gross wages and salaries already include overtime pay under the prompt, so PolicyEngine taxes $85,209 rather than double-counting the overtime premium." -us,scenario_026,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable payroll_tax value or explanation. This is a missing-output failure rather than a substantive payroll tax computation. -us,scenario_026,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated employee payroll tax as zero because it confused payroll tax with a net additional tax liability or employer-covered amount. The spouse's wages generate employee Social Security and Medicare taxes regardless of income-tax offsets, employer coverage, or household modesty." -us,scenario_026,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the $21,208 employer-sponsored insurance premium from the spouse's $85,209 wages when determining school-meal income. The applicable income remains approximately $85,209, producing an FPG ratio of 2.20, above the 185% reduced-price threshold; categorical eligibility also does not apply." +us,scenario_026,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model correctly applied federal Social Security and Medicare taxes to the $85,209 wage amount but then invented a North Carolina employee payroll tax. North Carolina individual income tax is not a mandatory employee-side payroll tax included in this output, so only the two federal FICA components belong in the total." +us,scenario_026,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model added the separately reported $7,746 FLSA overtime premium to annual gross wages and then deducted the head’s $21,208 employer-sponsored insurance premiums from the spouse’s FICA base. Gross wages already include overtime, and the head’s premium amount does not reduce the spouse’s $85,209 payroll-tax wage base." +us,scenario_026,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model added the $7,746 FLSA overtime premium to the spouse’s $85,209 gross wages even though the prompt states that gross wages are annual totals including overtime pay. Social Security and Medicare taxes apply to the $85,209 wage base, not $92,955." +us,scenario_026,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model deducted the head’s $21,208 employer-sponsored insurance premiums from the spouse’s wages when determining the FICA base. The spouse’s payroll taxes are calculated on the full $85,209 of wages because that premium input belongs to a different person and is not specified as the spouse’s pre-tax payroll deduction." +us,scenario_026,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model named the correct $85,209 wage base and the correct 6.2% and 1.45% federal rates but submitted $6,519.966 instead of carrying out that stated computation. The traced Social Security and Medicare components total $6,518.47." +us,scenario_026,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The submitted $10,618 is incompatible with applying 6.2% Social Security and 1.45% Medicare tax to the sole earner’s $85,209 wages, despite the model claiming that method. That wage base produces $6,518.47, with no Additional Medicare Tax." +us,scenario_026,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model double-counted overtime by adding the separately listed $7,746 FLSA overtime premium to the $85,209 gross wage total. The prompt specifies that gross wages already include overtime, so FICA is imposed on $85,209 rather than $92,955." +us,scenario_026,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so the required output was missing." +us,scenario_026,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated employee payroll tax as though it could net to zero based on modest household income or employer coverage. Employee Social Security and Medicare taxes apply directly to the spouse’s $85,209 wages and are not eliminated by income-tax deductions, credits, or the head’s employer coverage." +us,scenario_026,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $21,208 employer-sponsored insurance premium from $85,209 of wages to produce $64,001, even though the school-meal income calculation in this case uses $85,209. That income is 220% of the federal poverty guideline, above the 185% reduced-price limit, and no categorical pathway applies." us,scenario_026,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_026,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_026,spouse_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_026,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model allowed only $500 per child instead of the applicable $1,500 per child, understating the child deduction by $3,000. Its submitted $2,882 also contradicts its own stated $2,206.51 calculation; the correct $52,300.52 taxable income at 3.99% yields $2,086.79." -us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly substituted a federal standard deduction and federal-taxable-income framework for North Carolina's $25,500 joint standard deduction and $4,500 child deduction, then invented a dependent credit. Its $5,238 submission follows none of its intermediate calculations; North Carolina taxable income is $52,300.52 and tax is $2,086.79." -us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the $130 traditional IRA deduction from AGI and overstated the child deduction as $7,500 instead of $4,500. It then submitted $3,041 despite computing figures near $2,101 and $1,972; the correct computation is $52,300.52 multiplied by 3.99%." -us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model stopped after the joint standard deduction and omitted the $4,500 North Carolina child deduction. It also used AGI of about $82,431 instead of $82,300.52, leaving taxable income overstated relative to the correct $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $21,500 joint standard deduction instead of $25,500 and a $9,000 child deduction instead of $4,500. Those offsetting errors produce total deductions $500 too large; the correct $30,000 deduction gives $52,300.52 taxable income and $2,086.79 of tax." -us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model added the $7,746 FLSA overtime premium to the $85,209 annual gross wages even though the prompt states that gross wages already include overtime. It also omitted the $4,500 child deduction and used 4.25% instead of the 2026 rate of 3.99%." -us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the $25,500 joint standard deduction but omitted the $4,500 child deduction. This overstated taxable income as $56,801 instead of $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model deducted the $21,208 employer-sponsored insurance premium from wages even though it is not an additional adjustment to the listed annual gross wages in this calculation. It also used a $6,000 child deduction instead of $4,500, driving taxable income down to $29,593 rather than $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $3,538 implies a taxable base of about $88,672 at the 3.99% rate, so the model failed to apply the deductions that reduce $82,300.52 of AGI to $52,300.52. The required $25,500 joint standard deduction and $4,500 child deduction produce $2,086.79." -us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $21,208 employer-sponsored insurance premium again from gross wages, reducing AGI to $61,093. It also used a $6,000 child deduction instead of $4,500; correct taxable income is $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a 3.74% rate instead of North Carolina's 3.99% rate for 2026 and omitted the $4,500 child deduction. It also invented a second $130 IRA deduction for the head despite the prompt listing only the spouse's contribution." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated deductions or credits as eliminating North Carolina taxable income. The specified deductions total $30,000, leaving $52,300.52 taxable at 3.99%, so tax before refundable credits remains $2,086.79." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The $4,875 estimate does not reflect the derivation from $82,300.52 of AGI through the $25,500 joint standard deduction and $4,500 child deduction. Those deductions leave $52,300.52 taxable, to which the 3.99% rate applies." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $1,957.29 corresponds to only about $49,055 of taxable income at 3.99%, rather than the correct $52,300.52. It therefore applied roughly $3,245 of unsupported additional deductions beyond the $30,000 total allowed here." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Although the model identified the 3.99% rate and both deduction categories, its $2,027 answer implies taxable income of about $50,802 instead of $52,300.52. It overstated the combined standard and child deductions by about $1,499." -us,scenario_026,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied a rough flat-rate estimate to wages without deriving North Carolina AGI and the full $30,000 of state deductions. The correct base is $52,300.52 after the joint standard and child deductions, not approximately $85,000 of wages." -us,scenario_026,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $4,500 child deduction and therefore taxed about $56,801 rather than $52,300.52. Applying 3.99% after both the $25,500 standard deduction and child deduction yields $2,086.79." -us,scenario_026,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,payroll_tax_base,False,"The model double-counted the $7,746 overtime premium by adding it to annual gross wages that already include overtime. It also omitted the $4,500 child deduction and used a 4.25% rate instead of 3.99%." -us,scenario_026,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so its response failed the required output contract." -us,scenario_026,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the full $85,209 of wages as AGI, omitting the adjustments that reduce AGI to $82,300.52, and it omitted the $4,500 child deduction. The correct taxable income is $52,300.52, not $59,709." -us,scenario_026,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated standard deductions and personal exemptions as reducing North Carolina taxable income to approximately zero. The actual $30,000 of deductions leaves $52,300.52 taxable and $2,086.79 due at 3.99%." -us,scenario_026,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model double-counted the $7,746 overtime premium even though it is already included in the $85,209 annual gross wages. It also omitted the retirement adjustments and $4,500 child deduction and applied 4.5% instead of the 2026 rate of 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model allowed only $1,500 of child deductions instead of the applicable $4,500, leaving taxable income $3,000 too high. Its submitted $2,882 also contradicts its own recomputed tax of about $2,207." +us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly started from federal taxable income, substituted a federal standard deduction, used a 4.75% rate, and invented a $2,716 North Carolina dependent credit instead of applying North Carolina's $25,500 standard deduction and $4,500 child deduction to federal AGI. Its final $5,238 is also disconnected from every intermediate amount in its reasoning." +us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the $130 traditional IRA deduction from AGI and overstated the child deduction as $7,500 rather than $4,500. It then submitted $3,041 despite calculating liabilities between $1,972 and $2,101, so its final value does not follow its own computation." +us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model omitted both the $130 deductible traditional IRA contribution and the $4,500 North Carolina child deduction. Those omissions inflated taxable income from $52,300.52 to about $56,931 and produced the excessive tax." +us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $21,500 joint standard deduction and a $9,000 child deduction instead of the applicable $25,500 and $4,500 deductions. The net $500 over-deduction reduced taxable income below $52,300.52 and understated the tax." +us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $7,746 FLSA overtime premium even though gross wages already include all overtime, omitted the $4,500 child deduction, and used 4.25% instead of the 2026 rate schedule. Its asserted retirement-contribution addback and jump from $2,743 to $3,350 have no applicable North Carolina computation behind them." +us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model omitted the $4,500 North Carolina child deduction. It therefore taxed $56,801 rather than $52,300.52." +us,scenario_026,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $21,208 employer-sponsored insurance premium from wages even though the trace's federal AGI is $82,300.52, and it overstated the child deduction as $6,000 instead of $4,500. These errors drove taxable income down to $29,593 rather than $52,300.52." +us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $3,538 implies substantially more taxable income than the correct $52,300.52 at North Carolina's rate. The generic explanation never applies the $25,500 joint standard deduction and $4,500 child deduction that determine the liability." +us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced AGI by the $21,208 employer-sponsored insurance premium and used a $6,000 child deduction instead of $4,500. It consequently taxed $29,593 rather than $52,300.52." +us,scenario_026,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model invented a second $130 traditional IRA deduction for the head, omitted the $4,500 child deduction, and used a 3.74% rate. North Carolina instead taxes $52,300.52 under the 2026 rate schedule." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated deductions or credits as eliminating the liability, but the $25,500 standard deduction and $4,500 child deduction leave $52,300.52 taxable. That positive taxable income produces $2,086.79 before refundable credits, not zero." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $4,875 estimate does not apply the North Carolina deductions and rate schedule to the traced AGI. Federal AGI of $82,300.52 falls to $52,300.52 after the $25,500 standard deduction and $4,500 child deduction, yielding $2,086.79." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model understated the taxable-income base while naming only the standard deduction and no child-deduction calculation. The required deductions leave $52,300.52 taxable, whose scheduled tax is $2,086.79 rather than $1,957.29." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $2,027 reflects an understated taxable-income base after the joint and child deductions. The applicable deductions total $30,000, leaving $52,300.52 and a liability of $2,086.79." +us,scenario_026,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied a rough flat-rate calculation to approximately $85,000 of wages instead of starting from $82,300.52 of federal AGI and subtracting $30,000 of North Carolina deductions. The correct taxable-income base is $52,300.52." +us,scenario_026,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $4,500 North Carolina child deduction. It taxed about $56,801 rather than $52,300.52." +us,scenario_026,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted the $7,746 overtime premium even though annual gross wages already include overtime, omitted the $4,500 child deduction, and used 4.25%. These errors produced $64,547 of taxable income instead of $52,300.52." +us,scenario_026,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $74,555 without a valid listed adjustment and overstated the child deduction as $7,500 rather than $4,500. The correct computation begins at $82,300.52 and leaves $52,300.52 taxable." +us,scenario_026,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_026,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used gross wages as AGI, omitting the $2,778 traditional 401(k) contribution and $130 traditional IRA deduction, and it also omitted the $4,500 child deduction. The correct AGI is $82,300.52 and the correct taxable income is $52,300.52." +us,scenario_026,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated standard deductions and personal exemptions as reducing North Carolina taxable income to approximately zero. The applicable deductions total $30,000, leaving $52,300.52 taxable and a positive $2,086.79 liability." +us,scenario_026,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted the $7,746 overtime premium already included in annual wages, omitted the pre-tax retirement adjustments and $4,500 child deduction, and used a 4.5% rate. Those mistakes inflated taxable income to $67,455 instead of $52,300.52." +us,scenario_026,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model used an unsupported $81,509 taxable-income figure, a 4.5% rate, and an invented $19.41 nonrefundable credit. North Carolina starts from $82,300.52 of federal AGI, subtracts $30,000 of state deductions, and taxes the resulting $52,300.52 under the 2026 schedule." us,scenario_026,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_027,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model did not submit a parseable value for the requested variable, so it failed the required output contract rather than making a substantive tax computation." -us,scenario_027,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own derivation reaches taxable income of $0, gross tax of $0, and no nonrefundable credits, which yields federal income tax before refundable credits of $0. Its submitted -$3,195 applies an unsupported negative credit or sign reversal to a pre-refundable-credit liability that cannot go below zero in this case." +us,scenario_027,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output, violating the required structured-output contract." +us,scenario_027,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model correctly derived zero taxable income, zero gross tax, and no nonrefundable credits, but submitted -3195 instead of the zero produced by its calculation. Federal income tax before refundable credits does not become negative here, and its explanation explicitly ends with a value inconsistent with its own derivation." us,scenario_027,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_027,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly treated age 68 plus SSDI as placing the head into Connecticut's aged/disabled Medicaid category and assumed the household's $30,435 income was within that pathway's thresholds. PolicyEngine evaluated the aged/disabled and other pathways and assigned medicaid_category NONE, so the head did not qualify through any Medicaid category." -us,scenario_027,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model assumed that the aged Medicaid pathway applies automatically for a 65+ Connecticut resident and that the couple's income satisfied that pathway. The correct computation evaluates the pathway-specific income tests and all other categories, finds no qualifying category, and sets head Medicaid eligibility to 0." -us,scenario_027,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly invoked Connecticut aged/blind/disabled, Medically Needy, and spend-down concepts as an eligibility pathway for this fact pattern, and it treated the spouse's blindness and disability as expanding the head's eligibility. PolicyEngine's trace shows SSI received is zero, MAGI category income is 1.41 times FPL, and no senior, disability, or other categorical route applies to the head." -us,scenario_027,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model reduced Medicaid eligibility to being age 65 or older and meeting unspecified income and asset criteria. The correct derivation does not grant eligibility from age alone: after applying Connecticut's category-specific tests, the head qualifies through no Medicaid pathway." -us,scenario_027,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model did not provide a parseable value or explanation for head_medicaid_eligible. This is a missing-output failure rather than a substantive Medicaid-rule calculation. +us,scenario_027,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model asserted that $30,435 falls within Connecticut's aged/disabled Medicaid income threshold without applying the actual financial eligibility rules. Age 68 only opens the aged pathway; it does not establish eligibility, and the head qualifies through no category." +us,scenario_027,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated the couple's composition and the head's age as sufficient to place $30,435 within Connecticut's aged Medicaid limits. The aged pathway still requires satisfaction of its financial test, which this head fails, and no alternative category applies." +us,scenario_027,head_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"The model invented higher asset disregards and a medically needy spend-down based on the spouse's blindness and disability, even though no qualifying spend-down facts or expenses sufficient to establish that pathway were provided. The head also has $48,000 in bank assets and qualifies through neither the aged/disabled route nor any other Medicaid category." +us,scenario_027,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model declared that the head met Connecticut's income and asset criteria without applying either test. Being at least 65 creates an aged-category eligibility pathway, but the supplied income and assets do not satisfy a Medicaid category, so the head is not eligible." +us,scenario_027,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for head_medicaid_eligible, violating the required output contract." us,scenario_027,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_027,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated failure of the ordinary net-income test as complete ineligibility. Connecticut categorical eligibility keeps the household eligible, and the negative standard benefit formula is floored at the two-person minimum allotment rather than zero." -us,scenario_027,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied a conventional SNAP resource limit despite Connecticut categorical eligibility through TANF non-cash assistance. The $48,000 bank balance therefore does not bar eligibility, and the eligible two-person household receives the minimum allotment." -us,scenario_027,snap,claude-opus-4.7,llm_error,asset_resource,False,The model incorrectly disqualified the household under the elderly/disabled federal resource limit. Connecticut's categorical-eligibility pathway eliminates that resource-test barrier and leads to the two-person minimum allotment. -us,scenario_027,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model recognized the elderly/disabled rules but incorrectly imposed an asset test on a categorically eligible Connecticut household. The $48,000 balance does not prevent SNAP, and the benefit calculation floors the award at the minimum allotment." -us,scenario_027,snap,claude-opus-5,llm_error,categorical_eligibility,False,The model treated net income above the ordinary limit as eliminating SNAP. It omitted Connecticut categorical eligibility and the minimum-allotment floor for an eligible two-person household. -us,scenario_027,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model calculated income and medical deductions but then imposed a $4,500 resource limit. Connecticut categorical eligibility removes that resource test, after which the household receives the two-person minimum allotment." -us,scenario_027,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model made the $48,000 bank balance categorically disqualifying under a federal resource cap. Connecticut's TANF non-cash categorical-eligibility pathway prevents that asset disqualification." -us,scenario_027,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,The model converted net income above 100% FPL into a zero award. It omitted categorical eligibility and the rule flooring the eligible two-person household's negative formula result at the monthly minimum allotment. -us,scenario_027,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly identified broad-based categorical eligibility but stopped when maximum allotment minus 30% of net income was negative. For an eligible two-person household, that result is replaced by the minimum allotment, not zero." -us,scenario_027,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"The model applied a general SNAP resource limit to the $48,000 balance. Connecticut categorical eligibility makes that resource limit inapplicable and preserves eligibility for the minimum allotment." -us,scenario_027,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model recognized categorical eligibility but incorrectly allowed the benefit formula to end at zero. SNAP guarantees the applicable minimum allotment to an eligible household of two when the ordinary formula is negative. -us,scenario_027,snap,gemini-3.5-flash,llm_error,period_annualization,False,"The model correctly applied categorical eligibility and the minimum-allotment rule but used a flat $23 for all 12 months. PolicyEngine applies monthly minimums of $23.84 and $24.37 across the relevant periods, which sum to $287.68." -us,scenario_027,snap,gemini-3.6-flash,llm_error,period_annualization,False,"The model found the correct eligibility pathway and minimum-benefit floor but annualized an obsolete rounded $23 monthly amount. The applicable monthly minimum changes from $23.84 to $24.37 during the year, producing $287.68." -us,scenario_027,snap,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable SNAP output or explanation. -us,scenario_027,snap,gpt-5.4-mini,llm_error,asset_resource,False,"The model treated the household's assets as disqualifying under typical SNAP rules. Connecticut categorical eligibility removes the asset barrier, and the income calculation then yields the minimum allotment rather than no benefit." -us,scenario_027,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The prompt explicitly fixes unlisted income, expenses, and shelter costs at zero, so additional inputs were not required. The model failed to apply categorical eligibility and the two-person minimum allotment after using those zero inputs." -us,scenario_027,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated income above the ordinary net limit or benefit phaseout as producing no payable SNAP. Connecticut categorical eligibility preserves eligibility, and the two-person minimum-allotment floor produces a positive annual benefit." -us,scenario_027,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model let the negative maximum-allotment-minus-contribution calculation produce zero. For this eligible two-person household, SNAP substitutes the statutory monthly minimum allotment." -us,scenario_027,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,The model treated net income above the ordinary two-person limit as dispositive. It omitted Connecticut categorical eligibility and the minimum allotment payable when the standard formula is negative. -us,scenario_027,snap,gpt-5.6-terra,llm_error,asset_resource,False,"The model incorrectly applied the elderly/disabled SNAP resource limit to the $48,000 bank balance. Connecticut categorical eligibility removes that resource test and allows the minimum allotment." -us,scenario_027,snap,grok-4.3,llm_error,asset_resource,False,"The model relied on typical asset limits and treated the SSDI income as an additional reason to deny SNAP. Connecticut categorical eligibility removes the asset barrier, and the eligible household receives the two-person minimum allotment despite the negative ordinary formula." -us,scenario_027,snap,grok-4.5,llm_error,asset_resource,False,"The model imposed an approximately $4,500 resource cap and made excess assets automatically disqualifying. Connecticut's categorical-eligibility pathway prevents that disqualification." -us,scenario_027,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model compared annual net income directly with the annual maximum allotment instead of calculating the monthly expected contribution and then applying the minimum-allotment floor. The negative formula result for this eligible two-person household becomes the monthly minimum, not zero." -us,scenario_027,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model correctly noted the absence of shelter costs but treated net income above 100% FPL as eliminating eligibility. Connecticut categorical eligibility and the two-person minimum-allotment rule still produce a positive benefit. -us,scenario_027,snap,kimi-k3,llm_error,categorical_eligibility,False,The model stopped at the ordinary net-income limit after applying the standard and medical deductions. It omitted categorical eligibility and the minimum allotment due to an eligible two-person household. -us,scenario_027,snap,minimax-m3,llm_error,asset_resource,False,"The model used an incorrect 200%-FPL comparison and independently imposed a $4,250 asset limit. The traced gross-income ratio is 1.44, and Connecticut categorical eligibility removes the resource-test barrier, leaving the household eligible for the minimum allotment." -us,scenario_027,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly stated that $2,536 monthly income exceeds both applicable gross and net thresholds and also imposed a resource limit. The household passes the traced gross test at 1.44 times FPL, while Connecticut categorical eligibility removes the asset barrier and the minimum-allotment rule supplies the benefit." +us,scenario_027,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated failure of the 100% FPL net-income test as disqualifying after acknowledging Connecticut broad-based categorical eligibility. Categorical eligibility preserves eligibility, so the negative ordinary formula is replaced by the one- or two-person minimum allotment, totaling $287.68 for the year." +us,scenario_027,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied the federal SNAP resource ceiling to $48,000 of bank assets even though Connecticut broad-based categorical eligibility removes that asset test for this household. It also failed to award the minimum allotment after the ordinary benefit formula fell below zero." +us,scenario_027,snap,claude-opus-4.7,llm_error,asset_resource,False,"The model made the elderly/disabled federal resource limit dispositive. Connecticut categorical eligibility eliminates that resource barrier, leaving the eligible two-person household entitled to the minimum allotment." +us,scenario_027,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model incorrectly applied an approximately $4,500 resource limit to a Connecticut household that satisfies broad-based categorical eligibility. The $48,000 bank balance therefore does not eliminate SNAP, and the household receives the two-person minimum allotment." +us,scenario_027,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated countable income above the net-income limit as a complete denial. Connecticut categorical eligibility preserves eligibility, and because 30% of net income exceeds the maximum allotment, the two-person minimum allotment applies." +us,scenario_027,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model correctly identified the income and deductions but then imposed the federal elderly/disabled asset ceiling on a categorically eligible Connecticut household. Broad-based categorical eligibility removes that asset test, and the resulting eligible household receives the minimum allotment." +us,scenario_027,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model denied SNAP solely because $48,000 exceeded a $4,250 resource ceiling. Connecticut broad-based categorical eligibility removes that resource ceiling for this household, so the minimum two-person allotment remains payable." +us,scenario_027,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model used net income above 100% FPL to terminate eligibility. The household's categorical eligibility survives that comparison, and the negative ordinary allotment calculation triggers the minimum benefit rather than zero." +us,scenario_027,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model correctly found that the maximum-allotment-minus-30%-of-net-income formula was nonpositive but incorrectly stopped at zero. For an eligible one- or two-person household, the statutory minimum allotment replaces that result." +us,scenario_027,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"The model applied a general SNAP resource limit to the $48,000 bank balance. Connecticut categorical eligibility removes the asset test here, and the eligible two-person household receives the minimum allotment." +us,scenario_027,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model recognized categorical eligibility and correctly found that the ordinary formula was negative, but it incorrectly converted that result to no award. Eligible one- and two-person households receive the minimum allotment even when 30% of net income exceeds the maximum allotment." +us,scenario_027,snap,gemini-3.5-flash,llm_error,period_annualization,False,"The model identified the minimum-allotment rule but held the monthly amount fixed at $23 for all 12 months. PolicyEngine applies the indexed monthly minimums of $23.84 and $24.37 across the relevant months, which sum to $287.68 rather than $276." +us,scenario_027,snap,gemini-3.6-flash,llm_error,period_annualization,False,"The model correctly applied broad-based categorical eligibility and the minimum-allotment rule but annualized a stale flat $23 monthly amount. The 2026 month-specific minimums are $23.84 and $24.37, producing $287.68 for the year." +us,scenario_027,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SNAP output or explanation, so it failed the required structured-output contract." +us,scenario_027,snap,gpt-5.4-mini,llm_error,asset_resource,False,"The model relied on typical asset rules and treated the $48,000 balance as disqualifying. Connecticut categorical eligibility removes the resource test for this household, and high net income reduces the award only to the minimum allotment." +us,scenario_027,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The prompt explicitly sets unlisted shelter costs and other numeric inputs to zero, so no additional facts were needed. The model failed to apply Connecticut categorical eligibility and the minimum-allotment rule after calculating with the supplied zero expenses." +us,scenario_027,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated net income above the ordinary phaseout point as eliminating the benefit. Categorical eligibility keeps the household eligible, and an eligible two-person household receives the minimum allotment when the standard formula is nonpositive." +us,scenario_027,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model concluded that a nonpositive net-income formula meant no allotment. It omitted the minimum-benefit rule for categorically eligible one- and two-person households, which yields $287.68 annually here." +us,scenario_027,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model used net income above the two-person limit to deny SNAP outright. Connecticut categorical eligibility preserves eligibility despite that net-income result, and the household receives the statutory minimum allotment." +us,scenario_027,snap,gpt-5.6-terra,llm_error,asset_resource,False,"The model applied the federal resource limit to the household's bank assets. Connecticut broad-based categorical eligibility removes that asset test, so the household remains eligible for the minimum allotment." +us,scenario_027,snap,grok-4.3,llm_error,asset_resource,False,"The model invoked typical SNAP asset limits instead of Connecticut's categorical-eligibility treatment. The $48,000 bank balance is not disqualifying under that pathway, and excess net income leads to the minimum allotment rather than zero." +us,scenario_027,snap,grok-4.5,llm_error,asset_resource,False,"The model treated an approximately $4,500 elderly/disabled resource ceiling as controlling. Broad-based categorical eligibility removes the resource test in this case, leaving the two-person household eligible for the minimum benefit." +us,scenario_027,snap,grok-build-0.1,llm_error,categorical_eligibility,False,The model compared annual net income directly with the annual maximum allotment and treated a negative formula result as zero. SNAP first determines categorical eligibility and then applies the minimum allotment for an eligible two-person household when the ordinary monthly calculation is nonpositive. +us,scenario_027,snap,inkling,llm_error,categorical_eligibility,False,"The model made the 100% FPL net-income test dispositive and treated the absence of children as relevant to denial. Connecticut categorical eligibility applies to this elderly/disabled two-adult household, and the minimum allotment is payable despite net income above that threshold." +us,scenario_027,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model correctly accounted for zero shelter costs and the limited medical deduction but then treated the 100% FPL net-income test as an absolute eligibility bar. Categorical eligibility preserves SNAP eligibility and converts the nonpositive formula amount to the two-person minimum allotment. +us,scenario_027,snap,kimi-k3,llm_error,categorical_eligibility,False,The model stopped after finding net income above the ordinary two-person limit. It omitted Connecticut categorical eligibility and the minimum-allotment rule that applies once the standard benefit calculation falls below zero. +us,scenario_027,snap,minimax-m3,llm_error,asset_resource,False,"The model used an incorrect gross-income ceiling and separately imposed a $4,250 resource limit. The household passes Connecticut's 200% FPL gross test at a 1.44 FPL ratio, and categorical eligibility removes the asset test, leaving the minimum allotment payable." +us,scenario_027,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model incorrectly said $2,536 monthly income exceeded both the applicable gross and net limits and also imposed a resource ceiling. The household passes Connecticut's categorical gross-income test, its assets are disregarded through categorical eligibility, and the negative ordinary formula yields the minimum allotment." +us,scenario_027,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly placed net income below the poverty guideline and consequently calculated a large positive allotment. After the standard and allowable medical deductions, monthly net income remains about $2,113–$2,118, so 30% of net income exceeds the two-person maximum allotment and only the $287.68 annual minimum is paid." us,scenario_027,spouse_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_027,spouse_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model excluded the head’s Social Security disability income from Medicaid MAGI merely because it was nontaxable for federal income-tax purposes, reducing household MAGI to $275. The applicable Medicaid MAGI is 1.41 times FPL, above the category limit, and blindness or disability does not independently create eligibility when no disability-based pathway is satisfied." -us,scenario_027,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model treated blindness or disability plus no income in the spouse’s own name as sufficient for Medicaid and disregarded the pathway-specific financial test. Connecticut does not grant Medicaid solely from those statuses; the spouse satisfies no eligibility category. -us,scenario_027,spouse_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model asserted that $30,435 was below Connecticut’s disabled-adult income limits without applying a defined eligibility pathway or its actual financial threshold. The spouse satisfies no aged, blind, or disabled pathway, while MAGI equals 1.41 times FPL and exceeds the MAGI-category limit." -us,scenario_027,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model converted the spouse’s blindness and disability into an aged/blind/disabled Medicaid pathway and labeled the household income modest without applying the pathway’s financial rules. The spouse qualifies through no Medicaid category, and the over-limit MAGI test does not supply an alternative route." -us,scenario_027,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model assumed that disabled/blind status under age 65 placed the spouse in HUSKY D or a disability-based category and then asserted that spousal counting left income within limits. HUSKY D is the MAGI expansion pathway, whose income test fails at 1.41 times FPL, and the spouse satisfies no separate disability-based category." -us,scenario_027,spouse_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"The model assigned the spouse to the ABD/HUSKY C pathway and incorrectly declared $48,000 below the relevant couple asset limit without applying the pathway’s actual financial eligibility rules. The spouse satisfies no ABD category, so neither disability nor the asserted asset comparison establishes Medicaid eligibility." -us,scenario_027,spouse_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model treated disability status alone as Medicaid eligibility. The spouse must qualify through a recognized pathway and satisfies none; the engine category is NONE. -us,scenario_027,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no spouse_medicaid_eligible value or explanation, violating the required output contract." -us,scenario_027,spouse_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model stated that household MAGI was well below Connecticut’s adult Medicaid limit, but the applicable MAGI is 1.41 times FPL. That exceeds the MAGI-category threshold, and disability does not furnish another satisfied pathway." -us,scenario_027,spouse_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model reduced Medicaid MAGI to the $275 of interest by excluding the head’s Social Security disability income and therefore placed the spouse below 138% FPL. The applicable MAGI calculation is 1.41 times FPL, so HUSKY D fails, and the spouse satisfies no other Medicaid category." -us,scenario_027,spouse_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated disability status alone as sufficient for under-65 Medicare eligibility. It failed to require the spouse’s own qualifying Social Security disability-benefit entitlement and waiting period, neither of which was listed." -us,scenario_027,spouse_medicare_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that disability status confers Medicare eligibility regardless of age. An under-65 spouse needs a qualifying Medicare pathway, including the required disability-benefit entitlement period; the prompt lists only disability status." -us,scenario_027,spouse_medicare_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model converted the spouse’s disability flag into receipt of disability benefits. No SSDI receipt or completed Medicare waiting period is listed for the spouse, and unlisted benefit receipt must be treated as false." -us,scenario_027,spouse_medicare_eligible,claude-opus-5,llm_error,categorical_eligibility,False,The model invented a disability-related benefit status for the spouse and used it as a Medicare pathway. The facts establish disability but not the spouse’s SSDI entitlement or completion of the required waiting period. -us,scenario_027,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model conflated constant disability status with 24 months of the spouse’s own disability-benefit entitlement and incorrectly used the head’s Social Security disability income to support the spouse’s eligibility. A disabled spouse does not inherit Medicare eligibility from the beneficiary’s SSDI, and the spouse has no listed qualifying benefit history." -us,scenario_027,spouse_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model fabricated that the spouse had received Social Security disability benefits for at least 24 months. The prompt lists no disability-benefit receipt for the spouse, so that under-65 Medicare pathway is not satisfied." +us,scenario_027,spouse_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model excluded the head's Social Security income from household MAGI merely because it was nontaxable for income-tax purposes, reducing Medicaid MAGI to $275 instead of the engine's 1.41 times FPL. It also invoked a disability pathway without applying its categorical and financial requirements; the spouse falls into category NONE." +us,scenario_027,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated blindness or disability as automatic Medicaid eligibility and tested only the spouse's individually listed income and assets. Connecticut requires qualification under a specific pathway with its applicable financial rules, and this spouse qualifies under none of them." +us,scenario_027,spouse_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model asserted that $30,435 was below Connecticut's disabled-adult income limits without identifying or applying the governing HUSKY C threshold and related rules. Disability alone does not supply eligibility, and the spouse qualifies under no Medicaid category." +us,scenario_027,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model inferred eligibility from disability, blindness, and a vaguely characterized 'modest' household income instead of applying a defined Connecticut Medicaid pathway. The spouse qualifies under no category, while the applicable MAGI measure is 1.41 times FPL and exceeds the MAGI limit." +us,scenario_027,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model assumed that blind or disabled status placed the spouse into a disability-based category and then asserted that spousal-counted income was within its limit without performing that pathway's test. The spouse is assigned category NONE, so neither HUSKY D nor a blind/disabled pathway establishes eligibility." +us,scenario_027,spouse_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"The model asserted that $48,000 was below Connecticut's HUSKY C couple asset limit and therefore treated the spouse as financially eligible for an ABD pathway. That resource conclusion and automatic ABD categorization are wrong; the spouse qualifies through no Medicaid pathway." +us,scenario_027,spouse_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated disability status alone as sufficient for Medicaid. The spouse must qualify through a specific Connecticut eligibility pathway, and the engine assigns category NONE." +us,scenario_027,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required submission contract." +us,scenario_027,spouse_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model stated that household MAGI was well below Connecticut's adult expansion limit, but the computed MAGI is 1.41 times FPL, above the applicable 138% FPL threshold. Disability does not create an alternative qualifying category here." +us,scenario_027,spouse_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model excluded Social Security from Medicaid MAGI and reduced household MAGI to the $275 interest amount. The engine's Medicaid MAGI is 1.41 times FPL, above the expansion threshold, and the spouse qualifies through no non-MAGI category." +us,scenario_027,spouse_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model equated being blind and disabled with SSI-related Medicaid eligibility and treated the spouse's lack of individually listed income as dispositive. The spouse receives no SSI and qualifies under no Medicaid category, so the SSI-related shortcut does not apply." +us,scenario_027,spouse_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,The model treated disability status alone as sufficient for under-65 Medicare eligibility. It ignored that the spouse has no stated SSDI entitlement or completed 24-month waiting period and therefore does not qualify at age 39. +us,scenario_027,spouse_medicare_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that disability status confers Medicare eligibility regardless of age. For an under-65 spouse, the listed disability must be accompanied by a qualifying Medicare entitlement pathway, such as the required SSDI entitlement history, which is absent." +us,scenario_027,spouse_medicare_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model converted the spouse's disability flag into receipt of disability benefits. The facts list no SSDI for the spouse and no completed waiting period, so the under-65 Medicare disability pathway is not established." +us,scenario_027,spouse_medicare_eligible,claude-opus-5,llm_error,categorical_eligibility,False,The model invented a disability-related benefit status for the spouse and treated it as sufficient for Medicare. Only the head has listed Social Security disability income; the spouse has neither age-based eligibility nor a stated qualifying disability-benefit entitlement. +us,scenario_027,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model improperly attributed the head's Social Security disability income to the spouse's Medicare pathway and treated a constant disability condition as proof of 24 months of SSDI entitlement. A spouse does not acquire Medicare from another household member's SSDI receipt, and the spouse's own qualifying benefit history is unlisted and therefore false." +us,scenario_027,spouse_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model fabricated that the spouse had received Social Security disability benefits for at least 24 months. No such receipt or duration is listed, so the 39-year-old spouse fails both the age and disability-entitlement routes." us,scenario_027,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no spouse_medicare_eligible output or explanation, violating the required output contract." -us,scenario_027,spouse_medicare_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated PolicyEngine’s disability flag as an independent Medicare-eligibility condition. For this 39-year-old spouse, disability alone does not establish the required qualifying disability-benefit entitlement and waiting period." +us,scenario_027,spouse_medicare_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated PolicyEngine's disability flag as an automatic Medicare-eligibility flag. At age 39, disability alone is insufficient without a stated qualifying Medicare entitlement such as the required SSDI benefit history." +us,scenario_027,spouse_medicare_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated blindness and disability as direct Medicare eligibility categories. Those conditions do not by themselves establish under-65 Medicare entitlement, and the spouse has no listed SSDI receipt or waiting-period completion." us,scenario_027,spouse_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,ssi,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_028,child1_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied an inapplicable Pennsylvania CHIP ceiling of approximately 314% FPL and concluded that income near $61,277 passed. Under the applicable 2026 engine threshold for this household, that income exceeds the CHIP limit, and the child has no Medicaid eligibility pathway." -us,scenario_028,child1_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model used a 314% FPL CHIP ceiling and an incorrect estimate of the household’s FPL percentage to treat the income test as satisfied. The applicable Pennsylvania CHIP income test rejects approximately $61,277, while Medicaid eligibility is also false with category NONE." -us,scenario_028,child1_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model compared an understated $60,010 income amount with an inapplicable 314% FPL ceiling. The trace uses household income of approximately $61,277, which exceeds the applicable Pennsylvania CHIP limit for this household in 2026." -us,scenario_028,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_chip_eligible. The required output was numeric 0 because the age-10 child fails Pennsylvania’s CHIP income test at approximately $61,277 and is not Medicaid-eligible." -us,scenario_028,child1_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model asserted that the household fell within Pennsylvania’s higher CHIP income range without applying the applicable 2026 income limit. Approximately $61,277 exceeds that limit, and the child cannot qualify through Medicaid because the engine category is NONE." -us,scenario_028,child1_chip_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model applied an inapplicable 314% FPL ceiling and used $60,010 rather than the trace’s approximately $61,277 household income. Under the applicable 2026 Pennsylvania threshold, the traced income fails the CHIP test, and Medicaid supplies no alternate eligibility pathway." -us,scenario_028,child1_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model treated an inapplicable approximately 319% FPL ceiling as Pennsylvania’s operative CHIP limit and therefore placed the child inside a supposed CHIP income band. The applicable threshold excludes the household at approximately $61,277, and the child is also not Medicaid-eligible." -us,scenario_028,child1_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model correctly used approximately $61,277 of household income but compared it with an inapplicable 138%-319% FPL CHIP range. That income exceeds the applicable Pennsylvania CHIP limit for this household in 2026, and the child has Medicaid category NONE." +us,scenario_028,child1_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied a generic Pennsylvania CHIP ceiling of about 314% FPL and concluded that income around $61,277 passed. Under the applicable 2026 engine threshold, that income exceeds the CHIP limit, so the age-10 child fails CHIP's income test." +us,scenario_028,child1_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model used a 314% FPL upper limit and understated the household's income-to-FPL ratio as about 190%. The applicable 2026 Pennsylvania CHIP income test rejects household income of approximately $61,277." +us,scenario_028,child1_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model treated 314% FPL as the controlling Pennsylvania CHIP ceiling and omitted child support when stating annual income as $60,010. The trace uses household income of approximately $61,277, which exceeds the applicable 2026 CHIP limit." +us,scenario_028,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no answer for child1_chip_eligible. The required output is 0 because the age-10 child fails Pennsylvania CHIP's 2026 income test at household income of approximately $61,277." +us,scenario_028,child1_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model asserted that the household fell within Pennsylvania's higher CHIP income range without applying the applicable 2026 income ceiling. Household income of approximately $61,277 exceeds that ceiling, so Child 1 fails the income condition." +us,scenario_028,child1_chip_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model applied a generic 314% FPL limit and used MAGI of $60,010, excluding the child-support amount included in the traced household income of approximately $61,277. The traced income exceeds Pennsylvania's applicable 2026 CHIP limit." +us,scenario_028,child1_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model used a generic 319% FPL upper threshold and classified the household at about 187% FPL. The applicable 2026 Pennsylvania CHIP income limit is lower than the household's approximately $61,277 income, so the child fails the independent CHIP income test." +us,scenario_028,child1_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model correctly used household income of $61,277 but imposed an incorrect CHIP range of 138%–319% FPL, including a claimed family-of-four ceiling near $99,500. Under the applicable 2026 Pennsylvania CHIP threshold, $61,277 is over the income limit." +us,scenario_028,child1_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model declared the child within Pennsylvania's CHIP income range without calculating against the applicable 2026 ceiling. Household income of approximately $61,277 exceeds that limit, so the income criterion fails." us,scenario_028,child1_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,child1_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_028,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly used an approximately 215% FPL threshold as Pennsylvania's Medicaid limit for children. At 1.82 times FPL, Child 1 exceeds the applicable Medicaid thresholds and qualifies through no Medicaid category." -us,scenario_028,child1_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated Medicaid with CHIP by treating Pennsylvania's higher CHIP ceiling, approximately 314% FPL, as establishing Medicaid eligibility. CHIP taking over above the Medicaid cutoff is a separate coverage pathway and does not make Child 1 Medicaid-eligible." -us,scenario_028,child1_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_medicaid_eligible, violating the required output contract." -us,scenario_028,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that the household was within Pennsylvania's child Medicaid income limit without applying the actual threshold. The engine-derived MAGI level is 1.82 times FPL, above every applicable Medicaid pathway threshold, with no alternative eligibility category." +us,scenario_028,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied an approximately 215% FPL threshold as Pennsylvania's Medicaid limit for children. The engine places this child at 1.82 times FPL but in no Medicaid category, so the model used a broader child-coverage threshold instead of the applicable Medicaid pathway thresholds." +us,scenario_028,child1_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model combined Medicaid and CHIP into a single “Medicaid-equivalent” eligibility category and applied the CHIP coverage ceiling of roughly 314% FPL to the Medicaid output. CHIP eligibility above the Medicaid limit does not make the child Medicaid eligible; the engine assigns medicaid_category NONE. +us,scenario_028,child1_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child1_medicaid_eligible, violating the required output contract." +us,scenario_028,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that the household income falls within Pennsylvania's child Medicaid limit without applying the actual Medicaid category tests. At 1.82 times FPL, the child qualifies through none of those pathways and receives medicaid_category NONE." +us,scenario_028,child1_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model treated the household income as below Pennsylvania's Medicaid threshold for children. The applicable category evaluation at 1.82 times FPL produces no Medicaid category, so its threshold comparison used the wrong coverage cutoff." us,scenario_028,child1_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_028,child1_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a WIC category for school-age children ages 6-18 and treated employer-sponsored insurance as supportive rather than irrelevant. WIC child eligibility does not extend to a 10-year-old child, so Child 1 is categorically ineligible regardless of the household's income comparison." -us,scenario_028,child1_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model did not provide a parseable answer for child1_wic_eligible. The required output was 0 because a 10-year-old child is outside WIC's child categorical age limit. -us,scenario_028,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 10 as inside WIC's income-eligible child age range. WIC child categorical eligibility ends before age 5, so Child 1 is categorically ineligible before any income test is applied." -us,scenario_028,child2_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,The model treated Pennsylvania’s approximate 314% FPL ceiling as the complete CHIP income test. PolicyEngine’s applicable income test fails for child2; being age 10 and Medicaid-ineligible does not independently establish CHIP eligibility. -us,scenario_028,child2_chip_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,The model compared its estimated household income only with a broad 314% FPL ceiling and declared eligibility. Child2 passes the age condition but fails the CHIP income criterion applied by PolicyEngine. -us,scenario_028,child2_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,The model reduced CHIP eligibility to being under age 19 and below an approximate statewide upper-income limit. The engine’s person-specific CHIP income test fails for child2 despite the child being age 10. -us,scenario_028,child2_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used income below 314% FPL as a sufficient condition for Pennsylvania CHIP. PolicyEngine’s applicable CHIP income criterion is not satisfied, while age 10 satisfies the age criterion." -us,scenario_028,child2_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model omitted child2_chip_eligible entirely, violating the required output contract. The required numeric answer was 0 because child2 fails PolicyEngine’s CHIP income criterion." -us,scenario_028,child2_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model asserted that the household fell within Pennsylvania’s higher CHIP income range without applying PolicyEngine’s actual CHIP income test. That test fails for child2; the child’s age does not overcome the income failure. -us,scenario_028,child2_chip_eligible,kimi-k2.6,llm_error,thresholds_rates,False,The model treated MAGI below 314% FPL as dispositive and therefore skipped the applicable engine income criterion. Child2 passes the age condition at age 10 but fails PolicyEngine’s CHIP income test. -us,scenario_028,child2_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model assumed that income above its estimated Medicaid limit and below an approximate 319% FPL CHIP ceiling automatically placed child2 in CHIP. PolicyEngine separately applies the CHIP income criterion, which child2 fails." -us,scenario_028,child2_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model invented a generic 138%–319% FPL CHIP income band and placed the household inside it. PolicyEngine’s applicable CHIP income test fails for child2, so this range shortcut produces the wrong eligibility result." +us,scenario_028,child1_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a WIC pathway for school-age children ages 6–18. WIC covers qualifying children only until age 5, so the 10-year-old is categorically ineligible regardless of the model's income-threshold calculation." +us,scenario_028,child1_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model provided no output or explanation for child1_wic_eligible, violating the required structured-output contract." +us,scenario_028,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly placed a 10-year-old within WIC's eligible child age range. Children qualify categorically only while under age 5, so Child 1 is not eligible." +us,scenario_028,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model misapplied WIC's child age limit by treating age 10 as within it. Because WIC child eligibility ends at age 5, the income test cannot make this child eligible." +us,scenario_028,child2_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model treated Pennsylvania’s approximately 314% FPL headline ceiling as the applicable CHIP income test. PolicyEngine’s age-specific 2026 income criterion is not met, so child2’s age and Medicaid ineligibility do not produce CHIP eligibility." +us,scenario_028,child2_chip_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,The model applied a general 314% FPL upper limit and concluded that being under that ceiling was sufficient. Child2 fails PolicyEngine’s applicable 2026 CHIP income criterion despite satisfying the age condition. +us,scenario_028,child2_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model reduced the determination to being under age 19 and at roughly 190% FPL. It omitted the applicable age-specific Pennsylvania CHIP income test, which child2 fails." +us,scenario_028,child2_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used 314% FPL as a single statewide cutoff for every child. PolicyEngine applies the relevant 2026 age-specific CHIP income criterion, and child2 does not satisfy it." +us,scenario_028,child2_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child2_chip_eligible, violating the required output contract." +us,scenario_028,child2_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model asserted that the household falls within Pennsylvania’s higher CHIP income range without applying PolicyEngine’s applicable 2026 income threshold. That income criterion fails for child2. +us,scenario_028,child2_chip_eligible,kimi-k2.6,llm_error,thresholds_rates,False,The model treated 314% FPL as the controlling CHIP limit and regarded income below it as sufficient. Child2 fails the age-specific 2026 Pennsylvania CHIP income criterion used by PolicyEngine. +us,scenario_028,child2_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model constructed a Medicaid-to-319%-FPL CHIP band and placed child2 inside it. That shortcut substitutes a headline upper ceiling for PolicyEngine’s applicable 2026 age-specific CHIP income test, which child2 fails." +us,scenario_028,child2_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,The model invented a 138%–319% FPL CHIP eligibility band and treated the household’s income as falling within it. PolicyEngine’s relevant 2026 CHIP income criterion is not that generic band and is not satisfied by child2. +us,scenario_028,child2_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model declared the child within Pennsylvania’s CHIP income range without calculating the applicable 2026 age-specific threshold. Under that criterion, child2 fails the income test and is not CHIP-eligible." us,scenario_028,child2_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,child2_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_028,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied an incorrect 215% FPL Pennsylvania Medicaid threshold to a 10-year-old. At the engine-derived 1.82 times FPL, child2 exceeds every applicable Medicaid threshold and has no qualifying Medicaid category; the higher CHIP ceiling does not alter Medicaid eligibility." -us,scenario_028,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model stated a 157% FPL Medicaid threshold but then declared a child at about 187% FPL eligible, reversing its own threshold comparison. The engine income level is 1.82 times FPL, above all applicable Medicaid limits, leaving child2 with no Medicaid category." -us,scenario_028,child2_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model counted a family of five even though the household contains one head and three children, and it included child support in MAGI despite treating the calculation as MAGI-based. It then conflated eligibility under Pennsylvania's higher-income CHIP program with Medicaid eligibility; child2's 1.82-times-FPL MAGI produces no Medicaid category." -us,scenario_028,child2_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no answer for child2_medicaid_eligible, violating the required output contract." -us,scenario_028,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that household income falls within Pennsylvania's child Medicaid limit without applying the actual pathway thresholds. At 1.82 times FPL, child2 exceeds every applicable Medicaid income threshold and therefore has no qualifying Medicaid category." +us,scenario_028,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied an incorrect 215% FPL Medicaid threshold for Pennsylvania children. At the engine’s 1.82-times-FPL income level, Child 2 exceeds every applicable Medicaid threshold and receives no Medicaid category." +us,scenario_028,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model stated that income was about 187% FPL and that the children’s Medicaid limit was 157% FPL, then reversed its own threshold comparison by declaring the child eligible. Income above the cited limit fails that pathway, and the engine assigns no other Medicaid category." +us,scenario_028,child2_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated CHIP eligibility up to a higher income ceiling with Medicaid eligibility and treated qualification for either program as Medicaid qualification. It also counted a five-person family despite the household containing one head and three children; under the Medicaid calculation, Child 2 is at 1.82 times FPL and qualifies through no Medicaid pathway." +us,scenario_028,child2_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model omitted the required child2_medicaid_eligible output entirely, so it failed the submission contract rather than performing a substantive eligibility calculation." +us,scenario_028,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that household income was within Pennsylvania’s child Medicaid limit without applying the applicable thresholds. The engine places the child at 1.82 times FPL, above every available Medicaid income pathway, and assigns category NONE." +us,scenario_028,child2_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly treated 1.82 times FPL as below Pennsylvania’s Medicaid threshold for this 10-year-old. That income exceeds all applicable Medicaid pathways, leaving the child with category NONE." us,scenario_028,child2_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_028,child2_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model did not submit a parseable value for child2_wic_eligible, so it failed the output contract rather than applying the WIC income test. The correct computation denies eligibility because household income of about $61,277 exceeds the 185% FPL WIC limit for this household size." -us,scenario_028,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age as sufficient for WIC eligibility and skipped the household income screen. WIC requires income at or below 185% of the federal poverty line, and this household's income of about $61,277 exceeds the 2026 limit, so child 2 is not eligible." -us,scenario_028,child3_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied a 314% FPL statewide upper bound as the operative CHIP threshold and therefore treated $60,010 as qualifying income. The applicable computation uses approximately $61,277 of household income, which exceeds the relevant CHIP threshold, and child 3 has no Medicaid-linked eligibility because the child is not Medicaid-eligible." -us,scenario_028,child3_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model treated income near 190% FPL as automatically within Pennsylvania CHIP limits without applying the relevant CHIP income threshold. The applicable household income is approximately $61,277 and exceeds that threshold; child 3 is also not Medicaid-eligible." -us,scenario_028,child3_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used 314% FPL for a household of four as the controlling CHIP ceiling and understated relevant household income as $60,010. The computation uses approximately $61,277, which exceeds the applicable CHIP threshold, while child 3 is not Medicaid-eligible." -us,scenario_028,child3_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model omitted the required child3_chip_eligible output entirely, so it failed the submission contract rather than producing a substantive eligibility calculation." -us,scenario_028,child3_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model asserted that the household falls within Pennsylvania's higher CHIP income range without applying the applicable income threshold. Approximately $61,277 exceeds that threshold, and the child has no Medicaid-linked pathway because the child is not Medicaid-eligible." -us,scenario_028,child3_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model treated the interval above the child Medicaid limit and below an asserted 319% FPL CHIP ceiling as automatically CHIP-eligible. It therefore missed that approximately $61,277 exceeds the applicable CHIP threshold and that child 3's Medicaid eligibility category is NONE." -us,scenario_028,child3_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model placed $61,277 inside a generic 138%-319% FPL Pennsylvania CHIP range instead of applying the relevant CHIP income threshold. That income exceeds the applicable threshold, and child 3 is not Medicaid-eligible." -us,scenario_028,child3_chip_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model incorrectly treated Pennsylvania CHIP as having no income limit and assumed premiums preserve eligibility at every income level. The applicable computation imposes an income criterion that approximately $61,277 fails, and child 3 is not Medicaid-eligible." +us,scenario_028,child2_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable child2_wic_eligible output, violating the required submission contract." +us,scenario_028,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 10 as within WIC's child age category; WIC child eligibility ends at age five. It also failed to apply the 185%-of-poverty income ceiling, which this household's approximately $61,277 income exceeds." +us,scenario_028,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly stated that a 10-year-old is within WIC's child age limit, which ends at age five. It also incorrectly classified the household as income-eligible despite approximately $61,277 exceeding the applicable 185%-of-poverty ceiling." +us,scenario_028,child3_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model substituted a 314% FPL ceiling of about $100,951 for the applicable PolicyEngine 2026 Pennsylvania CHIP income threshold. Under the applicable test, approximately $61,277 exceeds the limit; age 10 does not overcome that failure, and Medicaid category NONE supplies no alternative pathway." +us,scenario_028,child3_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model treated an estimated income near 190% FPL as automatically within Pennsylvania's CHIP range without applying the applicable 2026 income threshold. Approximately $61,277 fails that income test, and child3 is not Medicaid-eligible." +us,scenario_028,child3_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a blanket 314% FPL CHIP ceiling and therefore classified $60,010 as qualifying income. The applicable PolicyEngine income test uses approximately $61,277 and places it above the Pennsylvania limit; Medicaid category NONE provides no other eligibility route." +us,scenario_028,child3_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for child3_chip_eligible. The required output was 0 because approximately $61,277 exceeds the applicable CHIP income threshold and child3 is not Medicaid-eligible." +us,scenario_028,child3_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model asserted that the household was within Pennsylvania's higher CHIP income range without applying the applicable 2026 threshold. Approximately $61,277 is above that threshold, while Medicaid category NONE supplies no qualifying pathway." +us,scenario_028,child3_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model imposed an approximate 319% FPL upper limit and treated income near 187% FPL as sufficient for CHIP. Under the applicable 2026 Pennsylvania rule, approximately $61,277 exceeds the CHIP income threshold, and being above the Medicaid limit does not itself establish CHIP eligibility." +us,scenario_028,child3_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model used an incorrect Pennsylvania CHIP band of 138%–319% FPL, with an asserted ceiling near $99,500. The applicable 2026 income test instead rejects approximately $61,277, and child3's Medicaid category is NONE." +us,scenario_028,child3_chip_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model incorrectly treated Pennsylvania CHIP as having no income limit and assumed premiums preserve eligibility at every income. PolicyEngine applies an income criterion that approximately $61,277 fails; age 10 qualifies only on age, and Medicaid category NONE supplies no alternative pathway." +us,scenario_028,child3_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model declared the child within Pennsylvania's CHIP income range without computing the applicable threshold. Approximately $61,277 exceeds that threshold, and the child is also not Medicaid-eligible." us,scenario_028,child3_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,child3_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_028,child3_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly treated Pennsylvania's higher CHIP-funded coverage threshold as a Medicaid eligibility threshold. Child3's MAGI of 1.82 times FPL exceeds the applicable Medicaid limits, leaving the child with Medicaid category NONE even if a separate CHIP pathway applies." -us,scenario_028,child3_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model mislabeled an approximately 215% FPL children's coverage threshold as Pennsylvania's Medicaid limit instead of separating Medicaid from CHIP eligibility. The Medicaid pathway evaluation at 1.82 times FPL yields no qualifying category. -us,scenario_028,child3_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model combined Medicaid and CHIP into a single coverage determination and concluded that CHIP eligibility implied Medicaid eligibility. It also counted child support in MAGI despite the prompt's separate income facts, but the decisive error is substituting CHIP's broader income range for the requested Medicaid determination." -us,scenario_028,child3_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child3_medicaid_eligible, violating the required output contract." -us,scenario_028,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that the household was within Pennsylvania's child Medicaid income limit without applying the actual pathway thresholds. At MAGI equal to 1.82 times FPL, child3 exceeds all applicable Medicaid limits and receives category NONE." +us,scenario_028,child3_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly treated a higher CHIP-funded coverage threshold of 215% FPL as a Medicaid threshold. Child3's 1.82-times-FPL MAGI exceeds the applicable Medicaid limits, and CHIP eligibility cannot be substituted for Medicaid eligibility." +us,scenario_028,child3_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model mislabeled an approximately 215% FPL children's coverage threshold as Pennsylvania Medicaid eligibility. The engine evaluates Medicaid separately and assigns child3 no Medicaid category at 1.82 times FPL. +us,scenario_028,child3_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model explicitly combined Medicaid and CHIP into one coverage determination even though the requested output was Medicaid alone. CHIP filling the income gap above Medicaid does not make child3 Medicaid-eligible, and child support was also improperly added to MAGI." +us,scenario_028,child3_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for child3_medicaid_eligible. The required output was 0 because child3 has no Medicaid eligibility category at 1.82 times FPL. +us,scenario_028,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that the household was within Pennsylvania's child Medicaid income limit without applying the applicable threshold. Child3's MAGI equals 1.82 times FPL, exceeds every available Medicaid pathway's income limit, and yields no eligibility category." +us,scenario_028,child3_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly placed child3's income below Pennsylvania's Medicaid threshold. At 1.82 times FPL, the income test fails across all Medicaid pathways and no Medicaid category applies." us,scenario_028,child3_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_028,child3_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model failed to provide a parseable answer for child3_wic_eligible. It did not complete the required eligibility determination or apply the WIC income screen to return 0. -us,scenario_028,child3_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the child's age as sufficient for WIC eligibility and ignored the separate household income test. Applying the 185% FPL WIC income limit to the household's roughly $61,277 annual income makes Child 3 income-ineligible, so the correct value is 0." -us,scenario_028,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model correctly stated that the available CTC could offset all income tax, then arbitrarily limited the nonrefundable CTC to $2,290 and left $2,284 unpaid. It also incorrectly included nontaxable child support in AGI and used the wrong deduction and filing-status brackets." -us,scenario_028,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model's reasoning explicitly derived zero after applying the nonrefundable CTC but submitted $2,818 instead. The $6,600 CTC supplies the full $3,949.20 needed to eliminate pre-credit tax, so no residual liability remains." -us,scenario_028,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly subtracted a predetermined maximum refundable ACTC from the total CTC before determining the nonrefundable credit used. The credit allocation instead uses $3,949.20 nonrefundably to eliminate income tax and assigns the remaining $2,650.80 to the refundable portion." -us,scenario_028,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model correctly concluded that the CTC eliminates the entire liability, then replaced zero with an unsupported $2,114 adjustment for credit ordering and rounding. Credit ordering and rounding do not prevent $3,949.20 of the $6,600 CTC from being used nonrefundably." -us,scenario_028,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used an obsolete $1,000-per-child nonrefundable CTC and therefore subtracted only $3,000. For 2026 each qualifying child generates $2,200, and $3,949.20 of the resulting $6,600 credit eliminates the full tax liability." -us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model's $1,395 residual contradicts its statement that $6,000 of child credits were applied to a smaller pre-credit tax amount. Under the applicable parameters, the total CTC is $6,600 and its $3,949.20 nonrefundable use reduces the liability to zero; it also used an incorrect $29,200 deduction." -us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed expiration of the governing 2026 provisions, reinstated personal exemptions, reduced the CTC to $1,000 per child, and used a roughly $12,000 standard deduction. The applicable calculation uses a $24,150 head-of-household standard deduction and a $2,200-per-child CTC, which eliminates all $3,949.20 of tax." -us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-expiration regime with personal exemptions, a reduced standard deduction, and a $1,000-per-child CTC. The applicable $24,150 standard deduction and $6,600 total CTC produce $3,949.20 of pre-credit tax and then eliminate it completely." -us,scenario_028,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract." -us,scenario_028,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The submitted $3,000 implies that the model failed to use enough of the nonrefundable CTC against the computed tax. Three qualifying children provide $6,600 of CTC, and $3,949.20 must be used nonrefundably to reduce the liability to zero." -us,scenario_028,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model included nontaxable child support in AGI and produced a residual exceeding the entire correctly computed $3,949.20 pre-credit tax, showing that it also failed to apply the available CTC. AGI is $60,010, taxable income is $35,860 after the $24,150 deduction, and the nonrefundable CTC eliminates the resulting tax." -us,scenario_028,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model stated that the credit reduces tax to zero but submitted $1,956, effectively treating only $2,184.20 as nonrefundable without a valid limitation. The nonrefundable CTC used equals the full $3,949.20 liability, while $2,650.80 remains refundable." -us,scenario_028,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The $2,890 answer is consistent with applying only part of the available child credit and leaving an unsupported residual. The $6,600 CTC permits $3,949.20 of nonrefundable use, eliminating the entire pre-credit tax." -us,scenario_028,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly applied an expired-law regime with a roughly $12,286 standard deduction, 15% bracket, and $1,000-per-child CTC. The applicable rules use the $24,150 head-of-household deduction, 2026 brackets, and $2,200 per child, leaving zero after $3,949.20 of nonrefundable CTC." -us,scenario_028,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract." -us,scenario_028,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly fixed the refundable ACTC at $5,100 first and treated only the remaining $900 as nonrefundable. The calculation instead uses $3,949.20 of the $6,600 CTC nonrefundably to eliminate tax, leaving $2,650.80 as the refundable portion." -us,scenario_028,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model used approximate EITC parameters that produced about $251 instead of $624.13 and calculated refundable CTC as about $2,665 instead of $2,650.80. It then discarded its own computed total of roughly $2,916 and submitted the $5,100 maximum ACTC cap as the entire answer." -us,scenario_028,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $1,267 of child support in MAGI even though child support received is not taxable income. It also invented a $3,850 ACTC amount: the 15% earned-income formula yields $8,625 and is not the binding limit; refundable CTC is the $2,650.80 unused CTC, and EITC adds $624.13." -us,scenario_028,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model prematurely set EITC to zero rather than applying the 2026 phaseout to obtain $624.13. It also used a $2,000-per-child CTC and approximate deduction and bracket values, understating the unused refundable CTC from $2,650.80 to about $2,037." -us,scenario_028,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly described refundable CTC as the unused credit after the nonrefundable offset but never carried that computation into its submitted $4,500 figure. It also eliminated the EITC instead of calculating the remaining $624.13 and failed to derive the $2,650.80 refundable CTC." -us,scenario_028,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model treated the $1,700-per-child ACTC ceiling as an automatic refundable payment. The ceiling is only a cap: refundable CTC is limited to the $2,650.80 of total CTC left after the nonrefundable offset, and the still-positive EITC is $624.13." -us,scenario_028,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used approximate 2025-era deduction, bracket, and $2,000-per-child CTC values, producing only $1,830 of unused CTC instead of $2,650.80. It also applied an incorrect EITC phaseout endpoint and reduced the $624.13 EITC to zero." -us,scenario_028,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model used approximate tax and CTC inputs, then replaced the unused-CTC calculation with an unsupported $2,000 estimate. It also treated the EITC as fully phased out, omitting the $624.13 remaining credit." -us,scenario_028,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The assertion that tax liability exceeds the CTC skips the nonrefundable-credit computation: $2,650.80 of CTC remains refundable after the tax offset. The model also prematurely phases out the EITC, omitting $624.13." -us,scenario_028,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model understated unused CTC as $66 instead of $2,650.80 because it used the wrong total CTC or pre-credit tax calculation. Its approximate EITC of $524 also understates the 2026 phaseout result of $624.13." -us,scenario_028,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The zero answer omits both refundable pathways. Applying the nonrefundable CTC first leaves $2,650.80 refundable, and the three-child EITC phaseout leaves another $624.13." -us,scenario_028,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model approximated the EITC phaseout as $574 instead of $624.13. More importantly, it incorrectly concluded that the CTC was fully used against tax liability, omitting the $2,650.80 refundable remainder." -us,scenario_028,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a $3,000 total CTC rather than the applicable $6,600 total for three qualifying children, eliminating the $2,650.80 refundable remainder. It also incorrectly treated the EITC as fully phased out instead of retaining $624.13." -us,scenario_028,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $6,000 total CTC and an incorrect $4,089.20 nonrefundable offset, yielding $1,910.80 rather than the $2,650.80 refundable CTC. It also omitted the $624.13 EITC entirely." -us,scenario_028,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for federal_refundable_credits. -us,scenario_028,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated the full $2,000-per-child CTC as refundable without first applying the nonrefundable credit against income tax or enforcing the refundable-credit rules. The correct refundable CTC is $2,650.80, and the omitted EITC is $624.13." -us,scenario_028,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model assumed that the income level eliminated every refundable credit. The CTC computation leaves $2,650.80 refundable after the nonrefundable offset, while the EITC phaseout leaves $624.13." -us,scenario_028,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Although the model used the correct $6,600 total CTC, it understated the nonrefundable CTC used against tax as $2,184.20, inflating refundable CTC far above $2,650.80. It also incorrectly set the EITC to zero rather than $624.13." -us,scenario_028,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model correctly obtained the $2,650.80 refundable CTC but incorrectly declared the EITC unavailable. Applying the three-child 2026 phaseout adds $624.13, producing $3,274.93." -us,scenario_028,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model treated the $5,100 aggregate ACTC ceiling as the amount paid solely because earnings satisfied the phase-in formula. Refundable CTC is also limited by unused total CTC and equals $2,650.80; the model additionally omitted the $624.13 EITC." -us,scenario_028,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly eliminated both refundable credits at this income. The unused CTC is $2,650.80 and the EITC remains $624.13 after its phaseout." -us,scenario_028,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model's roughly $40 ACTC implies an incorrect total CTC or pre-credit tax calculation; the unused refundable CTC is $2,650.80. Its approximate EITC phaseout also yields $660 instead of the traced $624.13." -us,scenario_028,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $3,000 total CTC and therefore wrongly concluded that none remained refundable; the applicable total is $6,600 and leaves $2,650.80 refundable. Its projected EITC parameters also overstate the phaseout result as $739 rather than $624.13." -us,scenario_028,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for federal_refundable_credits. -us,scenario_028,federal_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model calculated the EITC correctly at $624.13 and used the correct $6,600 total CTC, but understated the nonrefundable CTC used against tax as $3,838.10. The correct offset leaves $2,650.80 refundable rather than $2,761.90." -us,scenario_028,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated the ACTC earned-income formula and $5,100 ceiling as sufficient to award the maximum, omitting the binding unused-CTC limit of $2,650.80. It also incorrectly reduced the EITC to zero instead of $624.13." -us,scenario_028,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model repeatedly substituted projected parameters, used a $6,000 total CTC, and obtained only $1,918.80 of refundable CTC instead of $2,650.80. Its final EITC estimate of $1,042 used inconsistent inflation assumptions rather than the applicable phaseout calculation yielding $624.13." +us,scenario_028,child3_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child3_wic_eligible, violating the required structured-output contract." +us,scenario_028,child3_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 10 as within WIC's child age range, although WIC child eligibility ends at age five. It also failed to apply the household income test, under which approximately $61,277 exceeds the 185%-of-poverty limit." +us,scenario_028,child3_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model misapplied both controlling conditions: a 10-year-old is outside WIC's under-five child category, and household income of approximately $61,277 exceeds the applicable WIC income limit. Its statements that the child met the age limit and the household fell within the income range are both incorrect." +us,scenario_028,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly included $1,267 of child support in AGI, used the wrong standard deduction and filing-status brackets, and then subtracted only $2,290 of CTC despite acknowledging enough total CTC to eliminate the liability. The nonrefundable CTC used is $3,949.20, reducing this output to zero." +us,scenario_028,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly reasoned that the available nonrefundable CTC fully offsets income tax and explicitly derived zero, but submitted $2,818 instead. It failed to carry its own completed credit calculation into the numeric output." +us,scenario_028,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly reserved a fixed $5,100 ACTC amount before determining how much CTC offsets tax, leaving an artificial $1,596 liability. The credit allocation instead uses $3,949.20 nonrefundably to eliminate tax and treats the remaining $2,650.80 as refundable." +us,scenario_028,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model explicitly calculated that nonrefundable CTC reduces the liability to zero, then replaced that result with an unsupported $2,114 adjustment for credit ordering and rounding. No ordering or rounding step restores tax after the CTC has offset the full $3,949.20." +us,scenario_028,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model applied only $1,000 per child as nonrefundable CTC, subtracting $3,000 from its estimated tax. For 2026 each qualifying child generates a $2,200 CTC, and $3,949.20 of the $6,600 total is used nonrefundably to eliminate the liability." +us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model claimed to subtract $6,000 of child credits from tax on $30,810 of taxable income but still returned $1,395, which is arithmetically incompatible with its explanation. The available nonrefundable CTC offsets all $3,949.20 of the correctly calculated tax, leaving zero." +us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration, restored personal exemptions, reduced the CTC to $1,000 per child, and used obsolete deduction rules. The applicable 2026 rules provide a $24,150 head-of-household standard deduction and a $2,200-per-child CTC, whose nonrefundable use eliminates the $3,949.20 tax." +us,scenario_028,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly applied post-TCJA-expiration rules, including personal exemptions, a roughly $12,350 standard deduction, and a $1,000-per-child CTC. Under the applicable 2026 parameters, the $6,600 total CTC supplies enough nonrefundable credit to eliminate the $3,949.20 liability." +us,scenario_028,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_028,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The submitted $3,000 implies that the model failed to apply enough of the nonrefundable CTC after computing taxable income. The household has $6,600 of total CTC, of which $3,949.20 offsets the entire pre-credit tax and leaves zero before refundable credits." +us,scenario_028,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly included child support in AGI and returned a liability exceeding the $3,949.20 tax produced after the correct head-of-household standard deduction, showing that it also failed to apply the usable CTC. Child support is nontaxable, and $3,949.20 of nonrefundable CTC eliminates the correctly computed tax." +us,scenario_028,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model computed $4,140.20 of tax and then stated that only $2,184.20 of CTC was needed to reduce it to zero, an arithmetic error that directly produced the $1,956 residual. Using the correct tax of $3,949.20, the same amount of the $6,600 CTC is applied nonrefundably and the residual tax is zero." +us,scenario_028,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The $2,890 answer reflects an incomplete application of the child credit, since the available $6,600 CTC exceeds the $3,949.20 income tax before credits. The nonrefundable portion used equals the full $3,949.20 liability, leaving zero." +us,scenario_028,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration and used a reduced standard deduction, 2017-style brackets, and a $1,000-per-child CTC. The applicable 2026 parameters yield $3,949.20 of pre-credit tax and a $6,600 CTC, with $3,949.20 used nonrefundably to reduce the output to zero." +us,scenario_028,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_028,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly treated the maximum refundable amount of $5,100 as automatically carved out first, leaving only $900 of CTC to offset tax. The calculation instead applies $3,949.20 of the $6,600 CTC nonrefundably to eliminate tax, with the remaining $2,650.80 classified as refundable." +us,scenario_028,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model's own intermediate work produced an unused-CTC amount plus a residual EITC, but it discarded that calculation and submitted the $5,100 per-child ACTC ceiling. It also understated the EITC instead of applying the 2026 phaseout to obtain $624.13." +us,scenario_028,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included nontaxable child support in AGI and then declared the EITC fully phased out; the applicable income is $60,010 and yields $624.13. Its $3,850 ACTC has no relationship to the stated 15% earned-income calculation and misses the $2,650.80 unused CTC." +us,scenario_028,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used an obsolete $2,000-per-child CTC and an incorrect standard deduction and tax calculation, reducing the refundable CTC below $2,650.80. It also treated the three-child EITC as fully phased out instead of retaining $624.13." +us,scenario_028,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model reasoned that only about $1,200 of CTC remained refundable and that EITC was zero, then submitted $4,500 without deriving that number. The correct components are $2,650.80 of refundable CTC and $624.13 of EITC." +us,scenario_028,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model treated the $5,100 ACTC cap as an automatic refundable award, ignoring that ACTC cannot exceed the CTC left after nonrefundable use; that remainder is $2,650.80. It also eliminated the residual $624.13 EITC." +us,scenario_028,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used approximate prior-law values for the standard deduction, brackets, $2,000 CTC, and EITC endpoint. The 2026 parameters produce $2,650.80 of refundable CTC and leave $624.13 of EITC." +us,scenario_028,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model asserted full EITC phaseout and then assigned an unsupported $2,000 refundable CTC after correctly noting that ACTC depends on unused CTC. Applying the 2026 rules yields $624.13 of EITC and $2,650.80 of refundable CTC." +us,scenario_028,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model incorrectly concluded that tax liability exhausts the CTC and that EITC is fully phased out. The CTC exceeds the liability by a refundable $2,650.80, and the EITC phaseout leaves $624.13." +us,scenario_028,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model understated the unused CTC as $66 through an incorrect CTC or pre-credit-tax calculation; the refundable remainder is $2,650.80. It also understated the phaseout result, which produces a $624.13 EITC." +us,scenario_028,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The zero answer omits both refundable pathways. The household receives $2,650.80 of refundable CTC after nonrefundable CTC use and $624.13 of residual EITC." +us,scenario_028,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model incorrectly treated the entire CTC as used against tax liability, omitting the $2,650.80 refundable remainder. Its EITC phaseout calculation also understates the correct $624.13 amount." +us,scenario_028,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a $3,000 total CTC instead of the applicable $6,600 total for three children, which erased the refundable CTC. It also applied the wrong EITC eligibility endpoint; $624.13 remains after phaseout." +us,scenario_028,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $6,000 total CTC and an incorrect $4,089.20 pre-credit tax amount, producing only $1,910.80 of unused CTC instead of $2,650.80. It omitted the $624.13 EITC entirely." +us,scenario_028,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for federal refundable credits. +us,scenario_028,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated the entire nominal CTC as refundable and used approximately $2,000 per child. Refundability is limited to the $2,650.80 unused CTC after nonrefundable application, and the total must also include $624.13 of EITC." +us,scenario_028,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required unspecified extra 'triggers' and therefore omitted both credits despite three qualifying children and stated earned income. Those facts generate $2,650.80 of refundable CTC and $624.13 of EITC." +us,scenario_028,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Although the model used the correct $6,600 total CTC, it understated the tax absorbed nonrefundably as $2,184.20, inflating refundable CTC above $2,650.80. It also incorrectly set EITC to zero instead of $624.13." +us,scenario_028,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model correctly identified the $2,650.80 refundable CTC but incorrectly declared EITC unavailable. The three-child EITC phaseout leaves $624.13, which must be added." +us,scenario_028,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model treated the $1,700-per-child refundable ceiling as the award without limiting it to unused CTC, which is $2,650.80. It also omitted the $624.13 EITC." +us,scenario_028,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly concluded that income eliminates all refundable credits. The EITC retains $624.13 after phaseout, and the CTC leaves $2,650.80 refundable after offsetting tax." +us,scenario_028,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model's roughly $40 ACTC reflects an incorrect total CTC or tax-liability calculation; the unused refundable CTC is $2,650.80. Its approximate EITC also exceeds the trace amount of $624.13." +us,scenario_028,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an obsolete $3,000 total CTC and therefore eliminated ACTC, while the applicable $6,600 CTC leaves $2,650.80 refundable. Its projected EITC parameters also overstate the phaseout result, which is $624.13." +us,scenario_028,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model essentially recovered the refundable CTC but understated the EITC as about $340. Applying the 2026 three-child phaseout yields $624.13, bringing the total to $3,274.93." +us,scenario_028,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for federal refundable credits. +us,scenario_028,federal_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly calculated the $624.13 EITC but understated the nonrefundable CTC used as $3,838.10, inflating refundable CTC to $2,761.90. The correct refundable CTC remainder is $2,650.80." +us,scenario_028,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated the $5,100 ACTC ceiling as the refundable amount rather than limiting ACTC to the $2,650.80 of unused CTC. It also incorrectly eliminated the $624.13 residual EITC." +us,scenario_028,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model repeatedly mixed projected parameter sets, used a $6,000 total CTC, and settled on unsupported approximate tax and EITC figures. The applicable computation yields $2,650.80 of refundable CTC and $624.13 of EITC." +us,scenario_028,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly said the earned-income limitation produces a $3,200 refundable CTC; at these earnings that limitation is not binding, and unused CTC fixes the refundable amount at $2,650.80. It also omitted the $624.13 EITC." us,scenario_028,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_028,payroll_tax,claude-fable-5,llm_error,other,False,"The model derived every component correctly and explicitly summed $3,720 + $870 + $42 to $4,632, but submitted $5,730. Its numeric output contradicts its completed calculation." -us,scenario_028,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly set Pennsylvania employee payroll tax to zero and confused the employee unemployment-compensation contribution with unrelated state corporate taxes. Pennsylvania's 0.07% employee UC contribution adds $42 to the $4,590 federal FICA amount." -us,scenario_028,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model incorrectly stated that Pennsylvania has no mandatory employee payroll tax counted in this output. It omitted the 0.07% employee unemployment-compensation contribution, equal to $42 on $60,000 of wages." +us,scenario_028,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly derived $3,720 of Social Security tax, $870 of Medicare tax, and $42 of Pennsylvania employee unemployment contribution, totaling $4,632, but submitted $5,730. Its final value contradicts its own complete arithmetic." +us,scenario_028,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly set Pennsylvania's mandatory employee payroll contribution to zero. The 0.07% employee unemployment-compensation contribution adds $42 on $60,000 of wages to the $4,590 federal FICA total." +us,scenario_028,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model incorrectly stated that Pennsylvania has no mandatory employee payroll tax counted here. It omitted the 0.07% employee unemployment-compensation contribution, equal to $42 on $60,000." us,scenario_028,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,The model stopped after calculating federal Social Security and Medicare taxes. It omitted Pennsylvania's 0.07% employee unemployment-compensation contribution of $42. -us,scenario_028,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model incorrectly treated Pennsylvania's flat income tax as the state's only employee deduction and asserted that no separate employee payroll tax exists. Pennsylvania's employee UC contribution is 0.07% of wages, adding $42." -us,scenario_028,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model incorrectly asserted that Pennsylvania has no employee disability or unemployment contribution. The Pennsylvania employee UC contribution is 0.07% of $60,000, or $42." -us,scenario_028,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,The model calculated only federal Social Security and Medicare taxes. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution of $42. -us,scenario_028,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,The model limited payroll tax to federal Social Security and Medicare. It omitted the $42 Pennsylvania employee unemployment-compensation contribution on the head's wages. -us,scenario_028,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no payroll-tax value or explanation, so it failed the required output contract." -us,scenario_028,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,The model incorrectly asserted that no mandatory Pennsylvania employee payroll tax applies. The state's 0.07% employee UC contribution adds $42 to federal FICA. -us,scenario_028,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's stated Social Security and Medicare calculation equals $4,590, not the submitted $4,600. It also omitted Pennsylvania's $42 employee UC contribution, which raises the complete total to $4,632." -us,scenario_028,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,The model incorrectly excluded mandatory Pennsylvania employee payroll tax and counted only 7.65% federal FICA. Pennsylvania's 0.07% employee UC contribution adds $42. -us,scenario_028,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model recognized the Pennsylvania employee unemployment tax but calculated it as $7. Applying the 0.07% employee UC rate to $60,000 yields $42, making total payroll tax $4,632." -us,scenario_028,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,The model treated federal FICA as the entire payroll-tax output. It omitted Pennsylvania's mandatory employee UC contribution of $42. -us,scenario_028,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,The model applied only the 7.65% federal FICA rate. It omitted Pennsylvania's 0.07% employee unemployment-compensation contribution of $42. -us,scenario_028,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model incorrectly stated that no Pennsylvania employee payroll tax applies. Pennsylvania's 0.07% employee UC contribution adds $42 to the $4,590 federal total." +us,scenario_028,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,The model conflated the absence of a general state payroll tax with the absence of all mandatory employee contributions. Pennsylvania's employee unemployment-compensation contribution is 0.07% of wages and adds $42. +us,scenario_028,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,The model incorrectly asserted that Pennsylvania has no employee UI contribution. The mandatory 0.07% employee unemployment-compensation contribution adds $42 to federal FICA. +us,scenario_028,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,The model calculated only federal Social Security and Medicare taxes. It omitted Pennsylvania's 0.07% employee unemployment-compensation contribution of $42. +us,scenario_028,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,The model limited payroll tax to federal Social Security and Medicare. It failed to add Pennsylvania's mandatory employee unemployment-compensation contribution of $42. +us,scenario_028,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no payroll_tax output. It therefore failed the required structured-output contract. +us,scenario_028,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,The model incorrectly stated that no mandatory Pennsylvania employee payroll tax applies. Pennsylvania's 0.07% employee unemployment-compensation contribution adds $42 on the stated wages. +us,scenario_028,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model misadded its stated federal components: $3,720 of Social Security plus $870 of Medicare equals $4,590, not $4,600. It also omitted Pennsylvania's $42 employee unemployment-compensation contribution, so the complete total is $4,632." +us,scenario_028,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model incorrectly excluded all mandatory Pennsylvania employee payroll contributions. The state's 0.07% employee unemployment-compensation contribution adds $42 to the $4,590 FICA amount." +us,scenario_028,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model recognized Pennsylvania's employee unemployment contribution but calculated it as $7. The applicable 0.07% rate on $60,000 produces $42, yielding total payroll tax of $4,632." +us,scenario_028,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,The model treated federal FICA as the entire employee payroll-tax liability. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution of $42. +us,scenario_028,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model applied only the 7.65% federal FICA rate. It omitted Pennsylvania's 0.07% employee unemployment-compensation contribution, which adds $42." +us,scenario_028,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model explicitly but incorrectly excluded Pennsylvania employee payroll tax. The mandatory employee unemployment-compensation contribution is 0.07% of $60,000, or $42." us,scenario_028,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,The model stopped after federal Social Security and Medicare taxes. It omitted the $42 Pennsylvania employee unemployment-compensation contribution. -us,scenario_028,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll-tax value or explanation, so it failed the required output contract." -us,scenario_028,payroll_tax,kimi-k3,llm_error,payroll_tax_base,False,"The model incorrectly asserted that PolicyEngine applies no employee state payroll tax in Pennsylvania. PolicyEngine includes the state's 0.07% employee UC contribution, equal to $42." -us,scenario_028,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,The model incorrectly stated that Pennsylvania has no mandatory employee state payroll tax. It omitted the 0.07% employee unemployment-compensation contribution of $42. -us,scenario_028,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly asserted that Pennsylvania imposes no employee unemployment deduction beyond income-tax withholding. Pennsylvania's employee UC contribution is 0.07% of wages, adding $42 to federal FICA." -us,scenario_028,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model counted a five-person household even though the listed household contains one adult and three children. It then applied an inaccurate poverty threshold and rejected income that the school-meal calculation places at 1.82 FPG, below the 1.85 ceiling." -us,scenario_028,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model substituted an approximate $59,478 threshold for the applicable school-meal poverty-guideline calculation. The engine’s ratio is 1.82, so the income is below the 185% ceiling and produces the REDUCED tier." -us,scenario_028,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used an inaccurate projected 185% FPG cutoff of about $57,720. Under the applicable school-meal calculation, the household is at 1.82 FPG and therefore falls within the reduced-price range." -us,scenario_028,reduced_price_school_meals_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model estimated the household at roughly 190% FPG instead of applying the applicable school-meal poverty-guideline calculation. The computed ratio is 1.82, below the 1.85 reduced-price ceiling." -us,scenario_028,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model relied on an approximate and understated family-of-four income cutoff rather than the applicable school-meal guideline. The engine computes 1.82 FPG, placing the household below 185% and in the REDUCED tier." -us,scenario_028,reduced_price_school_meals_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly described one adult plus three children as a household of five and then contradicted its own stated threshold: $61,277 is below its approximate $66,000 ceiling. The applicable calculation yields 1.82 FPG and reduced-price eligibility." -us,scenario_028,reduced_price_school_meals_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an approximate $61,200 cutoff and treated the household as just above it. The applicable school-meal FPG calculation yields 1.82, below the 1.85 ceiling." -us,scenario_028,reduced_price_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,The answer implies that the model placed household income above 185% FPG without performing the applicable school-meal ratio calculation. That calculation yields 1.82 FPG and the REDUCED tier. -us,scenario_028,reduced_price_school_meals_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model asserted that $61,277 exceeds 185% FPG, but the applicable school-meal calculation places the household at 1.82 FPG. Because 1.82 is below 1.85, reduced-price support is positive." -us,scenario_028,reduced_price_school_meals_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied an inaccurate $57,720 family-of-four cutoff. The applicable school-meal guideline produces a ratio of 1.82, which satisfies the 185% ceiling." -us,scenario_028,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no answer or explanation for the requested variable. It therefore failed the required output contract rather than completing the reduced-price eligibility test. -us,scenario_028,reduced_price_school_meals_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model declared income above the reduced-price threshold without applying the school-meal FPG computation. The computed ratio is 1.82, so the household is below 185% FPG and enters the REDUCED tier." -us,scenario_028,reduced_price_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicit eligibility indicator as grounds for denial and skipped the income-based pathway. Categorical eligibility is false, but the independent income test succeeds at 1.82 FPG and produces reduced-price support." -us,scenario_028,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model placed the income above the four-person reduced-price threshold using an inaccurate threshold estimate. The applicable school-meal calculation yields 1.82 FPG, below the 1.85 limit." -us,scenario_028,reduced_price_school_meals_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used an estimated poverty threshold that placed $61,277 slightly above 185% FPG. The applicable school-meal computation instead gives a ratio of 1.82 and assigns the REDUCED tier." -us,scenario_028,reduced_price_school_meals_eligible,grok-4.3,llm_error,thresholds_rates,False,"The answer implies an incorrect comparison placing income above 185% FPG. The school-meal ratio is 1.82, so the income test grants reduced-price eligibility." -us,scenario_028,reduced_price_school_meals_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly treated $61,277 as exceeding the applicable family-of-four 185% FPG limit. The engine computes the school-meal ratio as 1.82, which is below the cutoff." -us,scenario_028,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model calculated an approximate ratio of 1.86 from the wrong poverty-guideline value. The applicable school-meal calculation yields 1.82, satisfying the 185% reduced-price limit." -us,scenario_028,reduced_price_school_meals_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model relied on projected poverty levels that placed the household above the cutoff. The applicable school-meal FPG ratio is 1.82, so the household qualifies for the REDUCED tier." -us,scenario_028,reduced_price_school_meals_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model used an understated poverty guideline, producing ratios of roughly 187% to 191%. The applicable school-meal calculation gives 1.82 FPG, below the 185% ceiling." -us,scenario_028,reduced_price_school_meals_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model applied an inaccurate approximate cutoff of $57,700. The applicable school-meal computation places the household at 1.82 FPG and therefore within the reduced-price range." -us,scenario_028,reduced_price_school_meals_eligible,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted the listed one adult and three children as a five-person household, then used a rough $59,000–$61,000 cutoff. The applicable calculation yields 1.82 FPG, below 185%, and assigns the REDUCED tier." +us,scenario_028,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model calculated federal FICA but did not include the mandatory Pennsylvania employee unemployment-compensation contribution. That state contribution equals $42 on $60,000 of wages." +us,scenario_028,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax output. It therefore failed the required structured-output contract. +us,scenario_028,payroll_tax,kimi-k3,llm_error,payroll_tax_base,False,"The model incorrectly stated that PolicyEngine applies no Pennsylvania employee state payroll tax. PolicyEngine includes the 0.07% employee unemployment-compensation contribution, equal to $42." +us,scenario_028,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,The model incorrectly treated Pennsylvania's mandatory employee state payroll tax as zero. The 0.07% employee unemployment-compensation contribution adds $42 to the federal payroll taxes. +us,scenario_028,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly claimed Pennsylvania has no employee unemployment deduction. The state's mandatory employee unemployment-compensation contribution is 0.07% of wages, producing $42 here." +us,scenario_028,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,The model treated the 7.65% federal Social Security and Medicare amount as the full payroll tax. It omitted Pennsylvania's mandatory employee unemployment-compensation contribution of $42. +us,scenario_028,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model counted five household members even though the listed household contains one head and three children, then asserted without valid threshold arithmetic that $61,267 exceeded the five-person limit. The applicable school-meal computation produces an income-to-FPG ratio of 1.82, within the 1.85 reduced-price ceiling." +us,scenario_028,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model added child support and taxable interest directly to wages and compared that improvised gross-income figure with an approximate guideline. The school-meal income computation yields a ratio of 1.82, which is below the 1.85 reduced-price limit." +us,scenario_028,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used an estimated 185%-FPG threshold of about $57,720 and concluded that wages alone exceeded it. The applicable 2026 school-meal calculation instead produces a 1.82 income-to-FPG ratio, placing the household in the REDUCED tier." +us,scenario_028,reduced_price_school_meals_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model estimated the household at roughly 190% of FPG rather than applying the school-meal ratio calculation. That calculation yields 182% of FPG, below the 185% ceiling." +us,scenario_028,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated wages, child support, and interest as a simple $61,277 school-meal income total and compared it with an approximate threshold. The applicable income computation yields a ratio of 1.82, so the household qualifies for reduced-price meals." +us,scenario_028,reduced_price_school_meals_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly called the one-adult, three-child household a household of five and then contradicted its own arithmetic by saying $61,277 exceeded an approximately $66,000 limit. The engine’s applicable ratio is 1.82, within the reduced-price range." +us,scenario_028,reduced_price_school_meals_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used $61,277 as gross school-meal income and compared it with an approximate $61,200 cutoff. The applicable school-meal income calculation produces a 1.82 FPG ratio, which does not exceed the 1.85 limit." +us,scenario_028,reduced_price_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer assumes that household income exceeds the reduced-price threshold without computing the applicable school-meal ratio. That ratio is 1.82, below the 1.85 ceiling." +us,scenario_028,reduced_price_school_meals_eligible,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model included child support in a $61,277 gross-income shortcut and declared that figure above 185% FPG. The applicable school-meal computation yields a ratio of 1.82 and therefore the REDUCED tier." +us,scenario_028,reduced_price_school_meals_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied an approximate reduced-price cutoff of $57,720 and concluded that $61,267 was too high. The applicable calculation places income at 1.82 times FPG, below the 1.85 cutoff." +us,scenario_028,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for the requested variable, so it failed the submission contract rather than completing the eligibility calculation." +us,scenario_028,reduced_price_school_meals_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model asserted that income was above the reduced-price threshold without applying the traced income-to-FPG calculation. The calculated ratio is 1.82, which qualifies under the 1.85 ceiling." +us,scenario_028,reduced_price_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly supplied eligibility indicator as evidence of ineligibility. Reduced-price eligibility is derived from the income test, and the household’s 1.82 FPG ratio places it in the REDUCED tier despite lacking categorical eligibility." +us,scenario_028,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model included reported child support in an undifferentiated household-income figure and treated that figure as exceeding the four-person limit. The applicable school-meal income calculation yields a ratio of 1.82, within the 1.85 ceiling." +us,scenario_028,reduced_price_school_meals_eligible,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model summed wages, interest, and child support to $61,277 and compared that shortcut with an estimated threshold. The school-meal calculation instead yields an income-to-FPG ratio of 1.82, so the household qualifies." +us,scenario_028,reduced_price_school_meals_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model asserted that income was above the threshold without calculating the applicable ratio. The household is at 1.82 times FPG, below the 1.85 reduced-price limit." +us,scenario_028,reduced_price_school_meals_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model treated $61,277, including child support, as the decisive school-meal income measure. The applicable computation produces a 1.82 FPG ratio and assigns the REDUCED tier." +us,scenario_028,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model estimated $61,277 as approximately 186% of FPG. The applicable school-meal calculation yields 182% of FPG, which is within the 185% reduced-price limit." +us,scenario_028,reduced_price_school_meals_eligible,inkling,llm_error,taxable_income_or_deductions,False,"The model compared a $61,277 total that included child support with an approximate $59,500 threshold. The applicable school-meal income calculation yields a ratio of 1.82 and therefore reduced-price eligibility." +us,scenario_028,reduced_price_school_meals_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model defined school-meal countable income as wages plus child support plus taxable interest and compared that sum with projected guidelines. The applicable computation yields a 1.82 FPG ratio, below the reduced-price ceiling of 1.85." +us,scenario_028,reduced_price_school_meals_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model estimated the household at roughly 191% of FPG and even placed wages alone near 187%. The traced school-meal calculation places the household at 182% of FPG, within the reduced-price range." +us,scenario_028,reduced_price_school_meals_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model used an approximate 185%-FPG cutoff of $57,700 and treated $61,277 as the applicable income. The actual school-meal ratio calculation is 1.82, below the 1.85 limit." +us,scenario_028,reduced_price_school_meals_eligible,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted five household members instead of the listed four and then used an approximate threshold range to classify $61,277 as excessive. The applicable school-meal calculation yields a 1.82 FPG ratio and the REDUCED tier." +us,scenario_028,reduced_price_school_meals_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted that income exceeded the reduced-price limit without performing the applicable ratio calculation. The household’s school-meal FPG ratio is 1.82, below the 1.85 ceiling." us,scenario_028,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,snap,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,ssi,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_028,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model included the $1,267 of child support in income and then replaced the actual Pennsylvania taxable income with an unsupported approximate $69,000 base. Pennsylvania taxable income for this trace is $60,010, and applying the 3.07% flat rate to that base gives $1,842.31, not $2,118." -us,scenario_028,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model identified the correct Pennsylvania taxable income base of $60,010 and the correct 3.07% flat rate, but it rounded the computed liability to $1,845 instead of carrying out the multiplication exactly. The correct computation is $60,010 × 0.0307 = $1,842.31." -us,scenario_028,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model did not provide a parseable value or explanation for this output. This is a missing-output failure rather than a substantive Pennsylvania tax calculation. -us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model used the correct Pennsylvania flat-rate framework but applied the 3.07% rate to an implied taxable-income base of about $53,746 instead of the trace's $60,010. It improperly reduced wages and interest through an unstated deduction or adjustment that does not apply to this Pennsylvania before-refundable-credit calculation." -us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model applied Pennsylvania tax to an implied base of about $113,355, nearly doubling the household's actual Pennsylvania taxable income. It treated unspecified deductions, credits, or liabilities as part of a broad estimate instead of applying Pennsylvania's 3.07% rate to the traced $60,010 taxable-income base." -us,scenario_028,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented a Pennsylvania head-of-household standard deduction of about $17,800 and subtracted it from the $60,010 income base. Pennsylvania's state income tax calculation here uses $60,010 of taxable income before forgiveness or refundable credits, so subtracting a federal-style standard deduction produced the understated $1,296 result." -us,scenario_028,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Pennsylvania has no refundable state EITC and considered only Tax Forgiveness. It omitted the 2026 Pennsylvania credit equal to 10% of the $624.13 federal EITC, which yields $62.41." -us,scenario_028,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model misclassified Pennsylvania's 2026 EITC as nonrefundable. The refundable credit equals 10% of the household's $624.13 federal EITC, producing $62.41." -us,scenario_028,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated Tax Forgiveness as the only relevant Pennsylvania credit and omitted the refundable state EITC. The household receives 10% of its $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly limited Pennsylvania refundable-credit analysis to Tax Forgiveness. Pennsylvania's separate 2026 refundable EITC equals 10% of the $624.13 federal EITC, yielding $62.41." -us,scenario_028,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,The model omitted Pennsylvania's refundable state EITC. Applying the 10% state rate to the household's $624.13 federal EITC gives $62.41. -us,scenario_028,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly stated that Pennsylvania has no state EITC. In 2026 the refundable Pennsylvania EITC is 10% of the federal credit, so $624.13 produces $62.41." -us,scenario_028,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model substituted the income test for Pennsylvania Tax Forgiveness for the separate state EITC calculation. The refundable Pennsylvania EITC equals 10% of the household's $624.13 federal EITC, producing $62.41." -us,scenario_028,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model evaluated only Pennsylvania Tax Forgiveness and stopped when income exceeded that program's threshold. It omitted the refundable state EITC of 10% of the $624.13 federal credit, which equals $62.41." -us,scenario_028,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no Pennsylvania refundable credit applied at this income. The household retains a $624.13 federal EITC, and Pennsylvania refunds 10% of it, or $62.41." -us,scenario_028,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted the household's eligibility for Pennsylvania's refundable EITC. The state credit is 10% of the $624.13 federal EITC, yielding $62.41." -us,scenario_028,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model focused on the Pennsylvania child and dependent care credit and treated missing childcare expenses as dispositive. It failed to calculate the separate refundable state EITC, which is 10% of $624.13 and equals $62.41." -us,scenario_028,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly denied eligibility for all Pennsylvania refundable credits. The household qualifies for a refundable state EITC equal to 10% of its $624.13 federal EITC, producing $62.41." -us,scenario_028,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model omitted Pennsylvania's refundable EITC eligibility pathway. Applying the state's 10% rate to the $624.13 federal EITC gives $62.41. -us,scenario_028,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, violating the required output contract." -us,scenario_028,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the absence of an explicit credit indicator in the household facts as eliminating Pennsylvania refundable credits. Eligibility follows from the computed $624.13 federal EITC, whose 10% Pennsylvania supplement is $62.41." -us,scenario_028,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separately listed refundable-credit indicator or lower-income signal. The household's computed federal EITC is $624.13, and Pennsylvania's refundable 10% credit therefore equals $62.41." -us,scenario_028,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the absence of childcare expenses as eliminating all Pennsylvania refundable credits. It omitted the state EITC, which depends on the $624.13 federal EITC and equals 10% of it, or $62.41." -us,scenario_028,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model omitted Pennsylvania's applicable refundable EITC. The credit is 10% of the household's $624.13 federal EITC, producing $62.41." -us,scenario_028,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no Pennsylvania refundable individual credit applied. The household qualifies for the state EITC equal to 10% of its $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model incorrectly used the income level and listed inputs to rule out every Pennsylvania refundable credit. The household still has a $624.13 federal EITC, generating a $62.41 Pennsylvania EITC at the 10% state rate." -us,scenario_028,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model omitted Pennsylvania's refundable state EITC. Ten percent of the household's $624.13 federal EITC is $62.41. -us,scenario_028,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable Pennsylvania personal-income-tax credit applied. The applicable state EITC is 10% of the $624.13 federal EITC, yielding $62.41." -us,scenario_028,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model explicitly and incorrectly asserted that Pennsylvania has no state EITC. Pennsylvania's refundable 2026 EITC equals 10% of the household's $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model evaluated Pennsylvania Tax Forgiveness and treated failure of its income test as eliminating all refundable credits. It omitted the separate state EITC equal to 10% of the $624.13 federal EITC, which produces $62.41." -us,scenario_028,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,"The model incorrectly stated that Pennsylvania has no refundable state EITC. The 2026 credit is 10% of the household's $624.13 federal EITC, yielding $62.41." -us,scenario_028,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,The model omitted the applicable Pennsylvania refundable EITC from its assessment of the state's limited credits. Applying the 10% state rate to the $624.13 federal EITC gives $62.41. -us,scenario_028,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Pennsylvania does not offer a refundable state EITC. In 2026 Pennsylvania refunds 10% of the household's $624.13 federal EITC, producing $62.41." +us,scenario_028,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model first included the $1,267 of nontaxable child support and then inexplicably inflated the Pennsylvania tax base to approximately $69,000. The correct base is $60,010, consisting only of $60,000 in wages and $10 in taxable interest." +us,scenario_028,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model identified the correct $60,010 base and 3.07% rate but rounded the resulting liability incorrectly. Multiplication yields $1,842.307, which rounds to $1,842.31 rather than $1,845." +us,scenario_028,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required output contract." +us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Although the model named the 3.07% rate and the taxable wage-and-interest categories, its $1,650 answer implies a tax base of only about $53,746. It omitted approximately $6,264 from the correct $60,010 Pennsylvania taxable-income base without identifying any valid Pennsylvania exclusion." +us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model applied an unspecified deduction-and-credit framework that produced $3,480, equivalent to taxing about $113,355 at Pennsylvania's 3.07% rate. The household's Pennsylvania taxable income is only $60,010, and no listed nonrefundable credit or deduction produces the submitted amount." +us,scenario_028,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly imported a federal-style $17,800 head-of-household standard deduction into Pennsylvania's personal income tax calculation. Pennsylvania provides no such standard deduction here, so the full $60,010 wage-and-interest base is taxed at 3.07%." +us,scenario_028,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model correctly computed 3.07% of $60,010 as $1,842.31, then improperly subtracted an invented $1.55 Pennsylvania Child and Dependent Care Tax Credit. No qualifying dependent-care expense was listed, and the traced computation retains the full $1,842.31 at this stage." +us,scenario_028,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model incorrectly asserted that Pennsylvania has no state EITC and considered only Tax Forgiveness. It omitted the refundable Pennsylvania EITC equal to 10% of the $624.13 federal EITC, which produces $62.41." +us,scenario_028,state_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model misclassified Pennsylvania's EITC as nonrefundable. The applicable refundable credit is 10% of the household's $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model restricted Pennsylvania relief to Tax Forgiveness and omitted the refundable state EITC. The household retains a $624.13 federal EITC after phaseout, generating a $62.41 Pennsylvania credit." +us,scenario_028,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model treated Tax Forgiveness as Pennsylvania's only relevant credit mechanism. It failed to apply the refundable Pennsylvania EITC at 10% of the $624.13 federal EITC, yielding $62.41." +us,scenario_028,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,The zero answer omits Pennsylvania's refundable EITC. Applying the 10% state percentage to the $624.13 federal EITC yields $62.41. +us,scenario_028,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model incorrectly stated that Pennsylvania has no state EITC. Pennsylvania refunds 10% of this household's $624.13 federal EITC, producing $62.41." +us,scenario_028,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model considered only the income limits for Tax Forgiveness and incorrectly denied the existence of a refundable Pennsylvania EITC. The state EITC equals 10% of the federal $624.13 credit, or $62.41." +us,scenario_028,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used the Tax Forgiveness threshold as if it exhausted Pennsylvania refundable-credit eligibility. It omitted the state EITC based on the household's $624.13 federal EITC, which produces $62.41 at the 10% state rate." +us,scenario_028,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model incorrectly concluded that no refundable Pennsylvania credit applies at this income. The household still receives $624.13 of federal EITC after phaseout, and Pennsylvania refunds 10% of it, or $62.41." +us,scenario_028,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,The zero answer omits the refundable Pennsylvania EITC pathway. Ten percent of the household's $624.13 federal EITC equals $62.41. +us,scenario_028,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model focused on the absence of child-care expenses and therefore missed the separate Pennsylvania EITC. That refundable credit depends on the $624.13 federal EITC, not child-care expenses, and equals $62.41 at the 10% state rate." +us,scenario_028,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model incorrectly denied eligibility for every refundable Pennsylvania credit. The household qualifies for a $624.13 federal EITC, and Pennsylvania's 10% credit produces $62.41." +us,scenario_028,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,The model omitted Pennsylvania's refundable EITC. Applying the state rate of 10% to the $624.13 federal EITC yields $62.41. +us,scenario_028,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for state_refundable_credits, violating the required submission contract." +us,scenario_028,state_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated the listed household facts as providing no basis for a Pennsylvania refundable credit, overlooking that wages, filing status, and three qualifying children generate a $624.13 federal EITC. Pennsylvania refunds 10% of that amount, or $62.41." +us,scenario_028,state_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model wrongly required a separate listed eligibility indicator and treated the income as too high for refundable credits. Federal EITC eligibility is computed from the supplied wages and three children, yielding $624.13 and therefore a $62.41 Pennsylvania EITC." +us,scenario_028,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model incorrectly made child-care expenses or another specially listed input necessary for a Pennsylvania refundable credit. The state EITC instead follows the computed $624.13 federal EITC and equals 10% of it, or $62.41." +us,scenario_028,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model omitted the refundable Pennsylvania EITC applicable to the household. The federal EITC is $624.13, so the 10% state credit is $62.41." +us,scenario_028,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model incorrectly concluded that no Pennsylvania refundable credit applies. It failed to calculate the state EITC as 10% of the $624.13 federal EITC, which equals $62.41." +us,scenario_028,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model treated the household's income as eliminating every Pennsylvania refundable credit. The federal EITC phaseout still leaves $624.13, on which Pennsylvania provides a 10% refundable credit of $62.41." +us,scenario_028,state_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The zero answer omits Pennsylvania's refundable state EITC. The credit is 10% of the household's $624.13 federal EITC, producing $62.41." +us,scenario_028,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model incorrectly stated that no refundable Pennsylvania personal-income-tax credit applies. The applicable Pennsylvania EITC is 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model explicitly and incorrectly asserted that Pennsylvania has no state EITC. The refundable state EITC equals 10% of the household's $624.13 federal EITC, yielding $62.41." +us,scenario_028,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model considered Tax Forgiveness and its income limit but omitted the distinct refundable Pennsylvania EITC. The state credit equals 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,kimi-k2.6,llm_error,credit_phaseout,False,"The model treated failure to meet Tax Forgiveness limits as eliminating all Pennsylvania refundable credits. It omitted the state EITC, which is 10% of the household's $624.13 federal EITC and equals $62.41." +us,scenario_028,state_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,The model incorrectly asserted that Pennsylvania has no refundable state EITC. Pennsylvania's 10% match of the $624.13 federal EITC produces a $62.41 refundable credit. +us,scenario_028,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,The model omitted the Pennsylvania refundable EITC from its limited-credit assessment. The household's $624.13 federal EITC generates a $62.41 state credit at the 10% rate. +us,scenario_028,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model explicitly and incorrectly denied the existence of a refundable Pennsylvania EITC. The state credit equals 10% of the household's $624.13 federal EITC, producing $62.41." +us,scenario_028,state_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,The model's zero answer omits Pennsylvania's refundable EITC. Applying the 10% state percentage to the $624.13 federal EITC yields $62.41. us,scenario_028,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_029,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $50,000 financial assistance amount in AGI, raising federal taxable income from $0 to $41,712 after its claimed standard deduction. It also applied an obsolete low single standard deduction and pre-TCJA bracket structure, but the decisive error is that PolicyEngine's taxable income calculation contains only the $312 of taxable interest, which is fully offset by the standard deduction." -us,scenario_029,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the $50,000 financial assistance entry as taxable federal income and added it to the $312 of taxable interest before subtracting the standard deduction. PolicyEngine's AGI includes only the $312 taxable interest, and that amount is below the single standard deduction, leaving zero taxable income and zero pre-refundable-credit federal income tax." -us,scenario_029,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $50,000 financial assistance in Medicaid MAGI, producing $50,312 instead of the $312 MAGI used for the ADULT expansion test. It also invoked resources for other pathways even though this MAGI-based category has no asset test." -us,scenario_029,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly stated that Oklahoma had not expanded Medicaid and therefore required pregnancy, disability, or another traditional category. Oklahoma's expansion supplies the ADULT pathway for this 21-year-old, and that MAGI-based pathway neither tests the listed assets nor disqualifies someone for employer-sponsored insurance." -us,scenario_029,head_medicaid_eligible,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $50,000 financial assistance to MAGI. Only the $312 taxable interest enters the traced MAGI, placing the head at 0.02 FPL and below Oklahoma's adult expansion limit." -us,scenario_029,head_medicaid_eligible,claude-opus-4.8,llm_error,other,False,"The model correctly derived eligibility from $312 of MAGI under Oklahoma expansion and explicitly said the head is eligible, but then submitted value = 0. Its numeric output directly contradicts its completed eligibility reasoning and the required 1-for-eligible contract." -us,scenario_029,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated the $50,000 financial assistance as Medicaid income. The ADULT-category MAGI is $312, not $50,312, so the expansion income limit is satisfied." -us,scenario_029,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the $50,000 financial assistance in MAGI and compared $50,312 with the expansion threshold. The traced MAGI contains only $312 of taxable interest, and employer-sponsored insurance does not remove Medicaid eligibility." -us,scenario_029,head_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly classified the $50,000 financial assistance as MAGI, causing it to fail the adult expansion income test. With $312 of MAGI, the head qualifies through Oklahoma's ADULT category without needing pregnancy, disability, or parent/caretaker status." -us,scenario_029,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model computed household MAGI as $50,312 by counting the $50,000 financial assistance. The expansion calculation uses $312, equal to 0.02 FPL, which satisfies the adult income limit." -us,scenario_029,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's above-threshold conclusion implies that it treated the $50,000 financial assistance as countable Medicaid income. The correct ADULT-category MAGI is $312, or 0.02 FPL, which is below Oklahoma's expansion threshold." -us,scenario_029,head_medicaid_eligible,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $50,000 financial assistance to taxable interest when computing Medicaid MAGI. The head's traced MAGI is $312, and employer-sponsored insurance is not an exclusion from the Oklahoma ADULT eligibility category." -us,scenario_029,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model overlooked Oklahoma's ACA ADULT expansion category and incorrectly required disability, pregnancy, a dependent child, or another traditional status. At age 21, the head meets the adult category, and MAGI of $312 satisfies its income test." -us,scenario_029,head_medicaid_eligible,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $50,000 financial assistance in the MAGI income test. The applicable MAGI is $312, placing the 21-year-old below Oklahoma's adult expansion limit." -us,scenario_029,head_medicaid_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model treated the listed $50,000 financial assistance as household income for adult Medicaid. The traced MAGI is only $312, so the expansion threshold is met." -us,scenario_029,head_medicaid_eligible,minimax-m3,llm_error,asset_resource,False,"The model treated the financial assistance and assets as grounds for failing Medicaid and also relied on employer-sponsored insurance. Oklahoma's MAGI-based ADULT pathway uses $312 of MAGI, imposes no asset test, and does not make employer coverage a bar to eligibility." -us,scenario_029,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the $50,000 financial assistance in Medicaid MAGI and therefore calculated $50,312 of countable income. The ADULT-category MAGI is $312, or 0.02 FPL, which falls below Oklahoma's expansion limit." -us,scenario_029,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model treated $312 of annual taxable interest as the only SNAP-relevant financial fact and ignored the household's bank and stock assets, which put a non-elderly, non-disabled Oklahoma SNAP unit over the resource limit. It then floored net income at zero and awarded a maximum allotment, so its $3,240 answer is the maximum-benefit shortcut for a financially eligible one-person unit rather than the result after the resource test." -us,scenario_029,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $50,000 financial assistance as Oklahoma taxable income and added it to the $312 of taxable interest. That inflated Oklahoma taxable income to $41,712 after the standard deduction and produced bracket tax that does not apply to the actual $312 income base." -us,scenario_029,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model computed Oklahoma tax from an adjusted gross income that included the $50,000 financial assistance instead of using only the $312 taxable interest income included by the tax calculation. After the standard deduction, the actual Oklahoma taxable income is zero, so the $1,827 answer reflects taxing excluded assistance income." -us,scenario_029,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly added $50,000 of financial assistance to Oklahoma gross income and thereby denied the Sales Tax Relief Credit. Only the $312 of taxable interest enters the applicable gross-income test, so the taxpayer receives the $40 credit for one exemption." -us,scenario_029,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model overlooked Oklahoma's refundable Sales Tax Relief Credit. At $312 of Oklahoma gross income, this taxpayer qualifies for the $40 amount attached to one exemption." -us,scenario_029,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model wrongly treated financial assistance and financial assets as income or resource disqualifiers for the Oklahoma Sales Tax Relief Credit. The credit uses $312 of gross income here and has no asset test, yielding $40 for one exemption." -us,scenario_029,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,The model incorrectly tied the Oklahoma Sales Tax Relief Credit to earned income and treated assets and financial assistance as barriers. The credit is available on $312 of qualifying gross income and equals $40 for the taxpayer's one exemption. -us,scenario_029,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,The model considered only earned-income and dependent-based pathways and failed to apply Oklahoma's Sales Tax Relief Credit. A single eligible taxpayer with $312 of gross income receives $40 without earned income or dependents. -us,scenario_029,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,The model stopped after finding no Oklahoma EITC and overlooked the refundable Sales Tax Relief Credit. The taxpayer's $312 of gross income satisfies its income test and one exemption produces a $40 credit. -us,scenario_029,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model wrongly concluded that Oklahoma offered no applicable refundable credit after discussing the EITC. It omitted the refundable Sales Tax Relief Credit, which equals $40 for this one-exemption taxpayer with $312 of gross income." -us,scenario_029,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model evaluated only Oklahoma's EITC and omitted the Sales Tax Relief Credit. No earned income is required for the latter, and $312 of gross income yields $40 for one exemption." -us,scenario_029,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model incorrectly included the $50,000 financial-assistance input in Oklahoma gross income. The Sales Tax Relief Credit instead tests $312 here, placing the taxpayer below the limit and producing $40." -us,scenario_029,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model omitted Oklahoma's refundable Sales Tax Relief Credit. The household has $312 of applicable gross income and one eligible exemption, so the credit is $40." -us,scenario_029,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model wrongly counted financial assistance as income under the Oklahoma Sales Tax Relief Credit test. Applicable gross income is $312, not $50,312, so the one-exemption credit is $40." -us,scenario_029,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,The model failed to apply Oklahoma's refundable Sales Tax Relief Credit. The $312 gross-income amount satisfies the limit and one exemption generates $40. -us,scenario_029,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model incorrectly treated age 65, disability, or a dependent as universally required for Oklahoma's Sales Tax Relief Credit. Those characteristics support the higher-income pathway; this taxpayer qualifies under the basic $20,000 gross-income limit with $312 and receives $40." -us,scenario_029,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,The model overlooked Oklahoma's refundable Sales Tax Relief Credit. The taxpayer's $312 of gross income qualifies one exemption for a $40 payment. -us,scenario_029,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model failed to identify the household facts supporting Oklahoma's Sales Tax Relief Credit. The relevant facts are $312 of gross income and one eligible exemption, which produce $40." -us,scenario_029,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model treated eligibility as requiring an expressly named credit indicator and did not derive the Oklahoma Sales Tax Relief Credit from the supplied income and filing facts. With $312 of gross income and one exemption, the refundable amount is $40." -us,scenario_029,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model incorrectly limited Oklahoma refundable credits to earned-income or similar bases. The Sales Tax Relief Credit requires neither earned income nor a dependent under this low-income pathway, and it pays $40 for one exemption at $312 of gross income." -us,scenario_029,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,The model wrongly required earned income or a dependent for every Oklahoma refundable credit. The Sales Tax Relief Credit applies to this single taxpayer based on $312 of gross income and yields $40. -us,scenario_029,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model overlooked the low-income pathway for Oklahoma's Sales Tax Relief Credit by requiring earned income, dependents, or another special fact. The taxpayer's $312 gross income and one exemption are sufficient for a $40 credit." -us,scenario_029,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model failed to recognize that the listed state, age, tax-unit size, and $312 of gross income establish the Oklahoma Sales Tax Relief Credit. One qualifying exemption produces a refundable $40 amount." -us,scenario_029,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model incorrectly required earned income, dependents, or special expenses for an Oklahoma refundable credit. The Sales Tax Relief Credit instead applies under the basic gross-income test and pays $40 for this one-exemption taxpayer." -us,scenario_029,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly treated a qualifying child as necessary for Oklahoma refundable credits. The taxpayer independently qualifies for the Sales Tax Relief Credit because gross income is $312, producing $40 for one exemption." -us,scenario_029,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,The model omitted Oklahoma's refundable Sales Tax Relief Credit. The applicable calculation is one exemption times $40 because gross income is only $312. -us,scenario_029,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model incorrectly treated being under 65, nondisabled, and without dependents as disqualifying and also relied on zero tax liability. Under the basic low-income pathway, $312 of gross income qualifies the taxpayer for the refundable $40 Sales Tax Relief Credit regardless of tax liability." -us,scenario_029,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model incorrectly added $50,000 of financial assistance to gross household income for the Oklahoma Sales Tax Relief Credit. The applicable gross income is $312, so the taxpayer qualifies under the basic income limit and receives $40." -us,scenario_029,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model made the fundamental factual error that Oklahoma has no state income tax. Oklahoma imposes an individual income tax and provides the refundable Sales Tax Relief Credit, which is $40 for this qualifying one-exemption taxpayer." -us,scenario_029,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model considered earned-income and child-related credits but omitted Oklahoma's Sales Tax Relief Credit. This credit does not require earnings or a child under the applicable low-income pathway; $312 of gross income and one exemption yield $40. -us,scenario_030,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $5,000 educational assistance and $12,000 financial assistance as taxable income and inflated AGI from $13,000 to $30,000. Applying the single standard deduction to that overstated AGI created $14,600 of taxable income and a fabricated regular tax liability; the correct computation leaves no taxable income after the standard deduction." -us,scenario_030,federal_refundable_credits,claude-fable-5,llm_error,age_disability,False,"The model calculated a phased-out childless EITC without applying the childless-worker minimum-age requirement. At age 23 with no qualifying child, the filer receives no EITC, so the refundable-credit total is $0." -us,scenario_030,federal_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"The model declared the 23-year-old eligible for the childless EITC and then substituted an erroneous maximum-credit figure. The childless-worker age restriction makes the filer ineligible before any phase-in, maximum, or phaseout calculation occurs." -us,scenario_030,federal_refundable_credits,claude-opus-4.7,llm_error,age_disability,False,"Although the model correctly excluded pregnancy as a qualifying child, it still computed a childless EITC for a 23-year-old. The filer fails the childless EITC minimum-age test, making the credit $0." -us,scenario_030,federal_refundable_credits,claude-opus-5,llm_error,age_disability,False,"The model explicitly recognized that the age requirement was not met and that the EITC was $0, but then contradicted that conclusion by assigning an approximately maximum childless EITC. PolicyEngine does not apply a childless phaseout amount after categorical age eligibility has failed." -us,scenario_030,federal_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"The model applied a minimum childless-EITC age of 19, carrying over the temporary expanded-age rule instead of the rule applicable in 2026. At age 23 and without a qualifying child, the filer is ineligible, so no phaseout calculation applies." -us,scenario_030,federal_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,The model treated the filer as an eligible childless worker and calculated a phaseout from the maximum credit. The filer is only 23 and fails the 2026 childless EITC age requirement; pregnancy does not supply a qualifying child. -us,scenario_030,federal_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"The answer assumes that low earnings alone establish eligibility for a childless EITC. The 23-year-old fails the childless-worker age test, and no qualifying child supports a different EITC pathway, yielding $0." -us,scenario_030,federal_refundable_credits,gpt-5.4-nano,llm_error,age_disability,False,"The model inferred EITC eligibility solely from the $13,000 wage level and omitted the childless-worker age test. Because the filer is 23 and has no qualifying child, the EITC and total refundable credits are $0." -us,scenario_030,federal_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"The model directly asserted that the 23-year-old qualifies for the childless EITC and proceeded to the income phaseout. The age restriction prevents childless EITC eligibility, so the phaseout formula must never be applied." -us,scenario_030,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly treated pregnancy as creating one dependent or qualifying child for EITC purposes. An unborn child is not a qualifying child, and the 23-year-old also fails the childless EITC age requirement, so the credit is $0." -us,scenario_030,federal_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model correctly recognized that pregnancy does not create a qualifying child but omitted the minimum-age restriction for the zero-child EITC. At age 23, the filer is ineligible, so the phase-in, maximum, and phaseout arithmetic is inapplicable." -us,scenario_030,head_chip_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated Texas CHIP Perinatal as direct CHIP eligibility for the pregnant head after applying an income limit. It missed that the head already qualifies for Medicaid under the pregnant category, which makes her ineligible for CHIP." -us,scenario_030,head_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated pregnancy as an age-independent CHIP eligibility category. PolicyEngine first establishes Medicaid eligibility under the pregnant category, and that Medicaid eligibility excludes the head from CHIP." -us,scenario_030,head_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model computed eligibility under a CHIP perinatal income threshold while omitting the Medicaid-precedence condition. Because the head qualifies for pregnancy Medicaid, she fails the requirement that a CHIP-covered person not qualify for Medicaid." -us,scenario_030,head_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated low income and pregnancy as sufficient for CHIP perinatal eligibility. It failed to apply the prior Medicaid determination: pregnancy Medicaid eligibility makes the head ineligible for CHIP regardless of being below a CHIP income limit. -us,scenario_030,head_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model recognized that the head qualifies for pregnancy Medicaid but then incorrectly converted that same status into CHIP perinatal eligibility. Medicaid and CHIP are not simultaneous pathways here; existing Medicaid eligibility categorically excludes the head from CHIP. -us,scenario_030,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated Texas's lack of ACA Medicaid expansion as excluding the head and failed to apply Texas's pregnant-individual Medicaid category. The head is pregnant, age 23, has MAGI at 0.60 FPL, and satisfies the immigration requirement, so the pregnancy category makes head_medicaid_eligible equal 1." -us,scenario_030,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model acknowledged pregnancy but then assumed no PolicyEngine eligibility trigger without applying the Texas pregnancy-category pathway. PolicyEngine's Medicaid rule classifies the head in the pregnant category and compares MAGI of 0.60 FPL to the Texas pregnant-individual threshold, yielding eligibility." -us,scenario_030,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable head_medicaid_eligible value or explanation. The required computation classifies the head as pregnant for Medicaid, verifies MAGI at 0.60 FPL and the immigration condition, and returns eligible." -us,scenario_030,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly required dependent children or ACA expansion eligibility for a pregnant adult in Texas and asserted that Texas lacks pregnancy-only Medicaid coverage. Texas's Medicaid pregnancy category covers pregnant individuals who meet the MAGI and nonfinancial tests, and this head's 0.60 FPL MAGI and immigration status satisfy those tests." +us,scenario_029,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $50,000 of financial assistance as taxable federal income, inflating AGI from $312 to $50,312. It also used an erroneous $8,600 single-filer standard deduction; the correct computation applies the applicable standard deduction to the $312 of taxable interest and leaves zero taxable income." +us,scenario_029,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $50,000 of financial assistance in federal taxable income. Only the $312 of taxable interest enters AGI, and the standard deduction reduces taxable income and federal income tax before refundable credits to zero." +us,scenario_029,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $50,000 financial-assistance input in Medicaid MAGI. Only the $312 of taxable interest enters the traced MAGI, placing the head at 0.02 FPL and within Oklahoma's ADULT expansion category." +us,scenario_029,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly stated that Oklahoma did not expand Medicaid and therefore required pregnancy, disability, or another traditional category. Oklahoma's expansion supplies the ADULT category for this 21-year-old, and that MAGI category has no asset test or employer-coverage exclusion." +us,scenario_029,head_medicaid_eligible,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $50,000 of financial assistance to Medicaid MAGI. The applicable MAGI is $312, or 0.02 FPL, so the adult expansion income test is satisfied." +us,scenario_029,head_medicaid_eligible,claude-opus-4.8,llm_error,other,False,"The model correctly derived that the $50,000 assistance and assets are excluded, the $312 MAGI is below 138% FPL, and the head is eligible through Oklahoma expansion, but then submitted value 0. Its numeric output directly contradicts its completed eligibility determination." +us,scenario_029,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated $50,000 of financial assistance as Medicaid income. The traced MAGI contains only $312 of taxable interest, making the head eligible under Oklahoma's adult expansion limit." +us,scenario_029,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the $50,000 financial assistance in MAGI and arrived at $50,312 rather than $312. Employer-sponsored insurance does not alter eligibility under the Oklahoma ADULT Medicaid pathway." +us,scenario_029,head_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the $50,000 financial assistance as MAGI. With traced MAGI of $312, the childless head qualifies directly through Oklahoma's adult expansion category and needs no pregnancy, disability, or caretaker pathway." +us,scenario_029,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly calculated ACA MAGI as $50,312 by including financial assistance. PolicyEngine's applicable MAGI is $312, equal to 0.02 FPL and below the adult expansion threshold." +us,scenario_029,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's above-threshold conclusion treats the listed financial assistance as countable Medicaid income. The correct MAGI derivation yields $312, or 0.02 FPL, which satisfies Oklahoma's adult expansion income limit." +us,scenario_029,head_medicaid_eligible,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $50,000 of financial assistance to the $312 of taxable interest for Medicaid MAGI. The assistance is excluded, and employer-sponsored insurance does not disqualify the head from the MAGI-based ADULT category." +us,scenario_029,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model overlooked Oklahoma's ACA ADULT category and incorrectly required disability, pregnancy, a dependent child, or another special status. At age 21, the head meets the adult category, and MAGI of 0.02 FPL satisfies its income test." +us,scenario_029,head_medicaid_eligible,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $50,000 financial assistance in the MAGI test. The traced MAGI is $312 rather than $50,312, placing the head below Oklahoma's expansion limit." +us,scenario_029,head_medicaid_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model treated the $50,000 financial-assistance amount as household income for Medicaid. It is excluded from the traced MAGI, leaving $312 and establishing eligibility under the adult expansion pathway." +us,scenario_029,head_medicaid_eligible,minimax-m3,llm_error,asset_resource,False,"The model wrongly treated the listed financial assistance and assets as disqualifying under Oklahoma's expansion pathway. MAGI-based ADULT Medicaid has no asset test, employer-sponsored insurance is not an exclusion, and the applicable income is only $312." +us,scenario_029,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly classified the $50,000 financial assistance as countable Medicaid income and calculated $50,312. The traced MAGI is $312, which is 0.02 FPL and below Oklahoma's adult expansion threshold." +us,scenario_029,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model overlooked that Oklahoma's ACA expansion itself provides the qualifying ADULT category for a 21-year-old nondependent. The head does not need disability, pregnancy, or another special basis, and the $312 MAGI satisfies the category's income test." +us,scenario_029,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model skipped the SNAP resource test and calculated an allotment solely from monthly interest income and the standard deduction. The household’s $54,985 of bank and stock assets exceeds Oklahoma’s applicable SNAP resource limit, making it ineligible before any maximum-allotment calculation." +us,scenario_029,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $50,000 of financial assistance to the $312 of taxable interest and treated the resulting $50,312 as Oklahoma adjusted gross income. Only the $312 of taxable interest enters the tax calculation, and the applicable deduction reduces Oklahoma taxable income to zero." +us,scenario_029,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The $1,827 answer reflects treating the listed $50,000 of financial assistance as taxable Oklahoma income before applying a standard deduction. The household instead has only $312 of specified taxable income, which is fully eliminated by the applicable deduction and produces zero tax." +us,scenario_029,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly added the $50,000 of financial assistance to gross income for the Oklahoma sales tax credit. The applicable gross income is $312, so the one-person unit receives the $40 refundable credit." +us,scenario_029,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,The model incorrectly asserted that Oklahoma has no generally applicable refundable credit. It omitted the $40 Oklahoma sales tax credit available to this one-person unit with $312 of gross income. +us,scenario_029,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model incorrectly counted the $50,000 financial assistance and the household’s assets against the Oklahoma sales tax credit. Neither enters the applicable gross-income calculation, which is $312 and produces a $40 credit." +us,scenario_029,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,The model wrongly treated earned income and substantial assets or assistance as barriers to all Oklahoma refundable credits. The sales tax credit is based on $312 of applicable gross income and yields $40 without an earned-income requirement. +us,scenario_029,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,The model reduced Oklahoma refundable credits to earned-income and dependent-based pathways and failed to apply the sales tax credit. A one-person unit with $312 of gross income and an age-21 head receives $40. +us,scenario_029,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model considered only earned-income and child-related credits after concluding that no EITC applied. It omitted Oklahoma’s refundable sales tax credit, which supplies $40 based on the unit’s $312 gross income." +us,scenario_029,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly concluded that the absence of earned income, dependents, disability, or old age eliminated every Oklahoma refundable credit. The Oklahoma sales tax credit independently provides $40 to this one-person low-income unit." +us,scenario_029,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model stopped after correctly finding no earned-income credit and failed to calculate the Oklahoma sales tax credit. With $312 of applicable gross income, that separate refundable credit equals $40." +us,scenario_029,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model incorrectly included the $50,000 financial assistance in income for the Oklahoma sales tax credit. The credit uses $312 of gross income here, placing the one-person unit within the limit and producing $40." +us,scenario_029,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The zero answer omits the Oklahoma sales tax credit. Applying its low-income rule to the one-person unit’s $312 gross income yields a $40 refundable credit. +us,scenario_029,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model incorrectly treated financial assistance as income under the Oklahoma sales tax credit’s threshold. Applicable gross income is only the $312 of taxable interest, so the credit is $40." +us,scenario_029,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,The model’s blanket finding that no Oklahoma refundable credit applies omits the sales tax credit. The one-person unit’s $312 gross income and age-21 head produce a $40 credit. +us,scenario_029,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model incorrectly required the head to be at least 65, disabled, or supporting a dependent for the Oklahoma sales tax credit. The age-21 head meets the modeled age requirement, and $312 of gross income yields the $40 credit." +us,scenario_029,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,The model incorrectly stated that Oklahoma offers no refundable credit to this household. It omitted the $40 sales tax credit generated from the one-person unit’s $312 gross income. +us,scenario_029,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,The model failed to identify the household facts supporting Oklahoma’s sales tax credit. The age-21 head and $312 of gross income qualify the one-person unit for $40. +us,scenario_029,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,The model treated eligibility as requiring an explicitly named credit indicator instead of applying Oklahoma’s sales tax credit rules to the listed facts. The one-person unit’s $312 gross income produces a $40 refundable credit. +us,scenario_029,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model wrongly required earned income or another specially listed credit basis. Oklahoma’s sales tax credit uses the unit’s low gross income, and $312 produces a $40 refundable amount." +us,scenario_029,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,The model incorrectly assumed that Oklahoma refundable credits require earned income or a dependent. The sales tax credit has neither requirement here and equals $40 for the one-person unit with $312 gross income. +us,scenario_029,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model incorrectly made earned income, dependents, or other special facts prerequisites for every Oklahoma refundable credit. It omitted the $40 sales tax credit determined from $312 of gross income." +us,scenario_029,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,The model failed to apply the Oklahoma sales tax credit to the facts already provided. An age-21 head in a one-person unit with $312 gross income receives $40. +us,scenario_029,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model incorrectly required earned income, dependents, or special expenses for any Oklahoma refundable credit. The low-income sales tax credit applies independently and equals $40." +us,scenario_029,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,The model wrongly treated qualifying children as necessary for Oklahoma refundable credits. The sales tax credit applies to this one-person unit based on $312 gross income and equals $40. +us,scenario_029,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,The model’s zero answer omits Oklahoma’s refundable sales tax credit. Applying the credit to the unit’s $312 gross income yields $40. +us,scenario_029,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model correctly ruled out an earned-income credit but incorrectly concluded that income and assets produced no Oklahoma low-income credit. Assets do not eliminate the sales tax credit, and $312 of gross income yields $40." +us,scenario_029,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model incorrectly required positive Oklahoma tax liability or age 65, disability, dependents, or earned income. The refundable sales tax credit applies to the age-21 head’s one-person unit at $312 gross income and equals $40." +us,scenario_029,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model incorrectly added the $50,000 financial assistance to gross household income for the sales tax credit. The relevant gross income is $312, so the household receives the $40 credit despite having no earned income." +us,scenario_029,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model falsely stated that Oklahoma has no state income tax and consequently skipped its refundable credits. Oklahoma’s sales tax credit contributes $40 for this one-person unit with $312 gross income. +us,scenario_029,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model considered only earned-income and child-related Oklahoma credits. It omitted the separate refundable sales tax credit, which equals $40 at $312 of gross income." +us,scenario_029,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model failed to recognize that the listed age, household size, and $312 gross income establish eligibility for Oklahoma’s sales tax credit. That credit contributes the full $40 state refundable amount." +us,scenario_030,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $5,000 of educational assistance and $12,000 of financial assistance in AGI, inflating it from $13,000 to $30,000. With only $13,000 of taxable wages, the single-filer standard deduction eliminates all taxable income, so applying the 10% and 12% brackets to $14,600 was erroneous." +us,scenario_030,federal_refundable_credits,claude-fable-5,llm_error,age_disability,False,"The model calculated a phased-out childless EITC without first applying the minimum-age requirement. At age 23, the filer is ineligible for the 2026 childless EITC, so the phase-in and phaseout arithmetic never applies." +us,scenario_030,federal_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"The model treated the 23-year-old as eligible for the childless EITC despite the minimum-age requirement. It also mislabeled $1,940 as the childless maximum and submitted that amount even though its own 7.65% calculation produced $994.50." +us,scenario_030,federal_refundable_credits,claude-opus-4.7,llm_error,age_disability,False,"The model correctly excluded pregnancy as a qualifying child but failed to enforce the childless EITC minimum age. Because the filer is 23, no childless EITC is available and the estimated phaseout calculation is irrelevant." +us,scenario_030,federal_refundable_credits,claude-opus-5,llm_error,age_disability,False,"The model explicitly recognized that the age requirement makes the childless EITC zero, then contradicted that rule by assigning an approximately maximum childless credit. PolicyEngine does not override the minimum-age test for this household." +us,scenario_030,federal_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"The model applied a minimum age of 19 to the 2026 childless EITC, carrying forward the temporary expanded-age rule instead of the applicable age threshold. The 23-year-old therefore fails eligibility before any phaseout calculation." +us,scenario_030,federal_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"The model assumed a 23-year-old qualifies for the childless EITC and proceeded directly to the income phaseout. The minimum-age test disqualifies the filer, and pregnancy neither changes filing status nor supplies a qualifying child." +us,scenario_030,federal_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"The answer implies that low earnings alone establish childless EITC eligibility and omits the age test. A 23-year-old with no qualifying child receives no EITC, and no other refundable credit applies." +us,scenario_030,federal_refundable_credits,gpt-5.4-nano,llm_error,age_disability,False,"The model inferred EITC eligibility solely from low wages without checking the childless claimant's minimum age. At age 23 and with no qualifying child, the filer receives zero EITC." +us,scenario_030,federal_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"The model stated that the 23-year-old qualifies for the childless EITC and calculated a phaseout amount. The filer fails the minimum-age condition, so no phaseout computation should be performed." +us,scenario_030,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly treated pregnancy as one qualifying child for EITC purposes. An unborn child is not a qualifying child for the credit, and the 23-year-old also fails the childless EITC minimum-age requirement." +us,scenario_030,federal_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model correctly recognized that pregnancy does not provide a qualifying child but then calculated the childless EITC without applying its minimum-age test. The filer is 23, so the credit is zero regardless of the phase-in cap and phaseout parameters." +us,scenario_030,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model inconsistently counted the unborn child as a qualifying child for EITC while excluding that same unborn child from CTC. Pregnancy does not create a qualifying child for either credit, and the filer cannot qualify through the childless EITC pathway at age 23." +us,scenario_030,head_chip_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model incorrectly treated Texas CHIP Perinatal as a CHIP eligibility pathway for the pregnant head based on income. It failed to apply Medicaid precedence: the head qualifies for pregnancy Medicaid, which makes her ineligible for CHIP." +us,scenario_030,head_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that pregnancy establishes adult CHIP eligibility regardless of age. The head is Medicaid-eligible under the pregnant category, and that Medicaid eligibility excludes her from CHIP." +us,scenario_030,head_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model calculated eligibility against the CHIP perinatal income limit but omitted the prior Medicaid-eligibility screen. Because the head qualifies for pregnancy Medicaid, CHIP eligibility is false regardless of whether her income falls below the perinatal threshold." +us,scenario_030,head_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated low income and pregnancy as sufficient for CHIP perinatal eligibility. It failed to exclude the head after determining that she qualifies for Medicaid under the pregnant-person category. +us,scenario_030,head_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model recognized that CHIP perinatal serves pregnancies not covered by Medicaid but then incorrectly assigned CHIP eligibility despite concluding that the head qualifies for pregnancy Medicaid. Medicaid eligibility takes precedence and makes the head ineligible for CHIP. +us,scenario_030,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated Texas’s non-expansion status as excluding the head from all Medicaid categories and omitted the state’s pregnancy category. Pregnancy is a qualifying category here, and MAGI income of 0.60 times FPL satisfies its financial test." +us,scenario_030,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed that no listed qualifier triggered eligibility even though the prompt explicitly states that the head is pregnant. It failed to apply Texas’s pregnant-individual pathway, under which 0.60-times-FPL MAGI and the satisfied immigration requirement yield eligibility." +us,scenario_030,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for the requested Medicaid indicator, violating the required output contract." +us,scenario_030,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,The model incorrectly imposed ACA expansion or dependent-child requirements on Texas pregnancy Medicaid and falsely stated that Texas lacks coverage for pregnant people without dependent children. The pregnancy category stands independently; this head’s 0.60-times-FPL MAGI and qualifying immigration status satisfy it. us,scenario_030,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_030,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_030,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model applied the correct employee Social Security and Medicare rates to the correct $13,000 wage base, but rounded the Medicare tax instead of keeping cents. Medicare tax is 1.45% of $13,000 = $188.50, not $189, so employee payroll tax is $806.00 + $188.50 = $994.50." +us,scenario_030,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model miscalculated Medicare tax: 1.45% of $13,000 is $188.50, not $189 or $189.90. It then compounded the arithmetic error by stating that $806 plus $189 equals $995.90; the exact component sum is $994.50." us,scenario_030,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_030,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the ordinary 130% FPL gross-income test as controlling and omitted Texas TANF non-cash categorical eligibility. That pathway preserves eligibility, and the resulting benefit is the one-person minimum allotment rather than zero." -us,scenario_030,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model focused on the ordinary gross-income test and never applied TANF non-cash categorical eligibility. It also incorrectly said $13,000 of wages could exceed the stated $22,424 threshold; the eligible household receives the minimum allotment." -us,scenario_030,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model considered only ordinary and BBCE gross-income limits and omitted the TANF non-cash categorical-eligibility pathway used here. Its net-income calculation is close to the trace, but that high net income triggers the minimum allotment for an eligible one-person household rather than ineligibility." -us,scenario_030,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded the entire $12,000 of financial assistance and therefore understated SNAP net income and expected contribution. Including the countable assistance makes the expected contribution exceed the maximum allotment, leaving only the monthly minimum benefit and an annual total of $287.68." -us,scenario_030,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model calculated the benefit from wages alone and omitted the countable $12,000 financial assistance from SNAP net income. With that assistance included, the expected contribution exceeds the maximum allotment and the household receives only the minimum benefit." -us,scenario_030,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model explicitly rejected categorical eligibility because the person did not receive cash TANF or SSI, overlooking eligibility through Texas TANF non-cash benefits. Consequently, it wrongly used the ordinary gross-income test to set SNAP to zero instead of applying the minimum allotment." -us,scenario_030,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $12,000 financial assistance from countable SNAP income and then produced a monthly benefit around $350, which exceeds its own stated one-person maximum of about $298. Correct countable income drives the expected contribution above the maximum, so the minimum allotment applies." -us,scenario_030,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income test and omitted TANF non-cash categorical eligibility. It also counted educational assistance in its $30,000 gross figure, while the trace excludes it; categorical eligibility leads to the minimum allotment." -us,scenario_030,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated $25,000 of wages and financial assistance as disqualifying under the ordinary 130% FPL test. It omitted TANF non-cash categorical eligibility, under which the household remains eligible and receives the minimum allotment." -us,scenario_030,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies a substantial benefit calculated from wages alone rather than the trace's full countable income. Including the $12,000 financial assistance makes the expected contribution exceed the maximum allotment, limiting the eligible household to $287.68 annually." -us,scenario_030,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model stopped at the ordinary 130% FPL gross-income test after counting wages and financial assistance. It omitted TANF non-cash categorical eligibility, which preserves eligibility and yields the one-person minimum allotment." -us,scenario_030,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model treated a 165% FPL gross-income threshold as a strict disqualifier and missed the household's TANF non-cash categorical eligibility. The household therefore remains eligible despite its gross-income ratio and receives the minimum allotment. -us,scenario_030,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model converted net income above the ordinary limit into zero eligibility. TANF non-cash categorical eligibility bypasses that disqualification, and an expected contribution above the maximum allotment triggers the statutory minimum benefit." -us,scenario_030,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted the unborn child as a second SNAP household member; pregnancy does not increase SNAP household size before birth. It also wrongly allowed the medical deduction for a 23-year-old who is neither elderly nor disabled, producing an inflated benefit instead of the one-person minimum allotment." -us,scenario_030,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model characterized the household as having very low countable cash income, which omits the $12,000 financial assistance included in SNAP income. The full countable-income calculation makes the expected contribution exceed the maximum allotment, so only $287.68 is paid annually." -us,scenario_030,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model gave zero without applying the TANF non-cash categorical-eligibility pathway or the minimum-allotment rule. The eligible one-person household receives a positive monthly minimum even though its expected contribution exceeds the maximum allotment. -us,scenario_030,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model correctly recognized that the expected contribution exceeds the one-person maximum but incorrectly converted that result into no benefit. Because TANF non-cash categorical eligibility is satisfied, the minimum allotment applies instead of zero." -us,scenario_030,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model treated income above the ordinary one-person gross-income limit as dispositive. It omitted TANF non-cash categorical eligibility and therefore failed to award the monthly minimum allotment. -us,scenario_030,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model correctly excluded educational assistance and counted wages plus financial assistance, but wrongly applied the ordinary Texas gross-income limit as a final bar. TANF non-cash categorical eligibility preserves eligibility, after which the minimum allotment is payable." -us,scenario_030,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model stopped after comparing $25,000 of countable gross income with the ordinary one-person limit. It omitted TANF non-cash categorical eligibility and the resulting minimum-allotment rule." -us,scenario_030,snap,grok-4.3,llm_error,categorical_eligibility,False,"The answer collapses the asset and income tests into zero despite assets of only $190 and TANF non-cash categorical eligibility. Once eligibility is applied, the high expected contribution produces the one-person minimum allotment rather than no benefit." -us,scenario_030,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model used the ordinary 130% FPL gross-income limit as an absolute eligibility test. It omitted TANF non-cash categorical eligibility, which leads to a positive minimum allotment despite the high expected contribution." -us,scenario_030,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model correctly excluded educational assistance but treated $25,000 of other income above 130% FPL as disqualifying. TANF non-cash categorical eligibility overrides that ordinary screening result, and the household receives the minimum allotment." -us,scenario_030,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so it failed the required output contract before any substantive SNAP computation could be evaluated." -us,scenario_030,snap,kimi-k3,llm_error,categorical_eligibility,False,The model treated monthly wages plus recurring financial assistance above the ordinary gross-income limit as disqualifying. It omitted TANF non-cash categorical eligibility and therefore failed to apply the monthly minimum allotment. -us,scenario_030,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model correctly settled on a one-person SNAP household but incorrectly treated annual post-earned-income-deduction income as exceeding a monthly net-income limit and ignored the standard deduction. More importantly, it omitted TANF non-cash categorical eligibility, under which the one-person minimum allotment remains payable." -us,scenario_030,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted both the countable $12,000 financial assistance and the standard deduction from its operative net-income calculation, then submitted $2,576 despite its own calculations yielding only $372–$384. The full trace produces an expected contribution above the maximum allotment, so the annual benefit is the sum of monthly minimum allotments, $287.68." +us,scenario_030,snap,claude-fable-5,llm_error,categorical_eligibility,False,The model applied the ordinary 130% FPL gross-income test as a disqualifying screen and omitted Texas TANF non-cash categorical eligibility at the 1.6 gross-income and 1.27 net-income ratios. Categorical eligibility leads to the one-person minimum allotment totaling $287.68. +us,scenario_030,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model denied SNAP based on total income and the ordinary gross-income limit without applying TANF non-cash categorical eligibility. It also incorrectly said $13,000 of wages could exceed a roughly $22,424 annual threshold, but the decisive missed pathway is categorical eligibility." +us,scenario_030,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model tested the household against ordinary and BBCE gross-income cutoffs but omitted its TANF non-cash categorical-eligibility pathway. Its own net-income calculation was close to the traced $1,653–$1,658 monthly amount, which triggers the minimum allotment rather than zero once eligibility is established." +us,scenario_030,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded the $12,000 financial-assistance input from SNAP income, producing an artificially low net income and a large formula benefit. The trace includes that assistance in the calculation, leaving net income near $1,653–$1,658 and limiting the eligible one-person household to the minimum allotment." +us,scenario_030,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model calculated the benefit from wages alone and omitted the countable $12,000 financial assistance. Including that assistance raises monthly net income to about $1,653–$1,658, so the computed benefit falls below the statutory minimum and $287.68 is paid across the year." +us,scenario_030,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model explicitly asserted that Texas lacked the categorical-eligibility route relevant here and therefore treated the ordinary 130% gross-income test as dispositive. The household qualifies through TANF non-cash categorical eligibility, after which its high net income produces the minimum allotment." +us,scenario_030,snap,claude-sonnet-5,llm_error,other,False,"The model's arithmetic is internally impossible: a $298 maximum minus an approximately $200 contribution is about $98, not $350, and a one-person benefit cannot exceed the stated maximum. It also omitted the $12,000 financial assistance, whose inclusion drives the formula benefit below zero and invokes the minimum allotment." +us,scenario_030,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model summed all $30,000 of listed wages, educational assistance, and financial assistance and applied the ordinary 130% FPL screen. It missed both the exclusion of educational assistance and the TANF non-cash categorical-eligibility pathway that prevents the ordinary gross-income test from denying benefits." +us,scenario_030,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated $25,000 of wages and financial assistance as an automatic failure of the ordinary one-person gross-income limit. It omitted TANF non-cash categorical eligibility, under which the household remains eligible and receives the minimum allotment." +us,scenario_030,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies a large benefit computed from the $13,000 earnings while omitting the countable $12,000 financial assistance. With that assistance included, net income is about $1,653–$1,658 monthly and the maximum-minus-contribution formula falls to the one-person minimum allotment." +us,scenario_030,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model correctly kept the unborn child out of SNAP household size but incorrectly made the ordinary 130% gross-income limit dispositive. TANF non-cash categorical eligibility establishes eligibility despite that screen, leaving a minimum annual allotment of $287.68." +us,scenario_030,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model treated its estimated 165% FPL gross-income cutoff as an absolute eligibility ceiling. The traced gross-income ratio is 1.6 and the household qualifies through TANF non-cash categorical eligibility, so the benefit proceeds to the minimum-allotment calculation." +us,scenario_030,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model equated net income high enough to erase the formula allotment with ineligibility and zero payment. Because the categorically eligible household satisfies the applicable tests, the one-person minimum allotment applies when 30% of net income exceeds the maximum allotment." +us,scenario_030,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted the unborn child as a second SNAP household member; pregnancy does not enlarge the SNAP assistance unit before birth. It also improperly allowed the elderly-or-disabled medical deduction for a 23-year-old without a listed disability, producing an erroneous positive formula benefit instead of the one-person minimum." +us,scenario_030,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model characterized countable cash income as very low and returned a large benefit, which omits the $12,000 financial assistance from the net-income calculation. Including it raises monthly net income to about $1,653–$1,658 and reduces the award to the minimum allotment." +us,scenario_030,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model assigned zero from household size and wages without applying TANF non-cash categorical eligibility or the minimum-benefit rule. A categorically eligible one-person household receives the minimum allotment even when its expected contribution exceeds the maximum allotment. +us,scenario_030,snap,gpt-5.5,llm_error,other,False,"The model correctly recognized that the expected contribution exceeds the one-person maximum allotment but incorrectly converted that result to zero. For this eligible one-person household, the SNAP minimum-allotment rule replaces the negative formula amount, yielding $287.68 across 2026." +us,scenario_030,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model treated income above the ordinary one-person gross-income limit as disqualifying. It omitted TANF non-cash categorical eligibility and therefore never applied the minimum-allotment rule. +us,scenario_030,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model correctly excluded educational assistance but wrongly made wages plus financial assistance above the ordinary gross-income limit dispositive. TANF non-cash categorical eligibility establishes eligibility at the traced income ratios, and the household then receives the minimum allotment." +us,scenario_030,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model denied eligibility solely because $25,000 exceeded the ordinary one-person gross-income limit. It omitted the household's TANF non-cash categorical-eligibility pathway and the resulting minimum benefit." +us,scenario_030,snap,grok-4.3,llm_error,categorical_eligibility,False,"The zero answer collapses the asset and income tests into ineligibility, even though $190 passes the asset test and TANF non-cash categorical eligibility resolves the income screening. The eligible one-person unit receives the minimum allotment totaling $287.68." +us,scenario_030,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model applied the 130% FPL gross-income limit as an absolute bar. It omitted TANF non-cash categorical eligibility, which allows the household to proceed to a minimum-allotment award." +us,scenario_030,snap,grok-build-0.1,llm_error,categorical_eligibility,False,The model correctly excluded educational assistance but incorrectly denied SNAP under the ordinary 130% gross-income test. TANF non-cash categorical eligibility governs this household and results in the minimum allotment. +us,scenario_030,snap,inkling,llm_error,categorical_eligibility,False,"The model incorrectly counted the $5,000 educational assistance in gross SNAP income and then applied the ordinary 130% FPL screen as dispositive. Educational assistance is excluded, and TANF non-cash categorical eligibility establishes eligibility despite the remaining income." +us,scenario_030,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so it failed the required structured-output contract." +us,scenario_030,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model treated $2,083.33 of monthly wages and financial assistance as an automatic failure of the ordinary one-person gross-income limit. It omitted TANF non-cash categorical eligibility and consequently the minimum-allotment calculation." +us,scenario_030,snap,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model correctly settled on a one-person SNAP unit but compared annual post-deduction income of $10,400 directly with a monthly net-income limit and omitted the standard deduction. It also ignored the categorical-eligibility and minimum-allotment rules that produce a positive benefit when the formula contribution exceeds the maximum." +us,scenario_030,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model initially computed only from wages, omitted both the countable $12,000 financial assistance and the standard deduction, and then submitted $2,576 despite its own calculations yielding $372–$384. Including all traced income and deductions produces net income near $1,653–$1,658 and the minimum allotment totaling $287.68." +us,scenario_030,snap,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted the unborn child as a second SNAP household member and reduced net income to zero without a valid deduction pathway. SNAP treats this as a one-person unit, and its countable income produces only the minimum allotment; the submitted $3,576 also contradicts its stated $4,272 calculation." us,scenario_030,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_030,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_031,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model recognized the age-related standard deduction but still left a small positive tax liability after deductions. It failed to carry the deduction calculation through: taxable IRA and pension income, reduced by the alimony expense where PolicyEngine deducts it, is below the single age-65-plus standard deduction, so taxable income is zero." -us,scenario_031,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an artificially low post-TCJA-reversion standard deduction estimate of $10,207 and explicitly disallowed the $1,165 alimony deduction. PolicyEngine deducts the alimony expense in AGI and applies the age-65-plus standard deduction, which eliminates the $13,243 AGI and leaves no taxable income." -us,scenario_031,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own calculation reaches taxable income of $0, tax of $0, and nonrefundable credits of $0, which supports a submitted value of 0. It violated the answer contract by submitting 1268.5 while its explanation says the correct federal income tax before refundable credits is zero." -us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used gross retirement income less alimony and compared it with an estimated FPL threshold. It omitted the senior-or-disabled pathway's deduction for health insurance premiums, including Medicare Part B, which reduces countable income enough to pass the applicable test." -us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model deducted only alimony from gross retirement income and treated the remaining amount as countable income. California's non-MAGI senior-or-disabled calculation also deducts health insurance premiums, including Medicare Part B, and the head passes both the resulting income test and the asset test." -us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model incorrectly described the Aged and Disabled program as MAGI-based and applied a roughly 100% FPL limit to gross retirement income. The applicable optional senior-or-disabled pathway is non-MAGI and deducts Medicare Part B and other health insurance premiums from countable income. -us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model compared the full $23,853 of retirement income with a broad 100%-138% FPL range. It failed to deduct health insurance premiums, including Medicare Part B, under the non-MAGI senior-or-disabled income test." -us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model repeatedly applied MAGI expansion and SSI-linked thresholds to a 67-year-old while treating the retirement income as the decisive measure. PolicyEngine instead uses California's optional non-MAGI senior-or-disabled category, where Medicare Part B and other health insurance premiums are deducted before testing income." -us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model substituted the SSI cash-benefit income limit for the optional senior-or-disabled Medicaid income test. SSI receipt is not required, and the applicable non-MAGI calculation deducts Medicare Part B and other health insurance premiums, leaving the head income-eligible." -us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,The model divided gross annual income into a monthly amount and compared it directly with an estimated 138% FPL limit. It omitted the non-MAGI deduction for Medicare Part B and other health insurance premiums that makes the head pass the senior-or-disabled income test. -us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model compared all $23,853 of retirement income directly with 138% FPL. The senior-or-disabled non-MAGI pathway deducts health insurance premiums, including Medicare Part B, before applying its income limit." -us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies a direct comparison of gross household income with a general Medi-Cal limit. The applicable senior-or-disabled calculation deducts Medicare Part B and other health insurance premiums from countable income, producing income eligibility." -us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated the full $23,853 as income for the Aged, Blind, and Disabled threshold. It omitted the non-MAGI health-insurance-premium deduction, including Medicare Part B, that brings countable income within the senior-or-disabled limit." -us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model compared monthly gross income with 138% FPL without performing the category's countable-income calculation. California's non-MAGI senior-or-disabled pathway deducts Medicare Part B and other health insurance premiums, after which the head passes the income test." -us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated $23,853 of annual retirement income as exceeding the aged Medi-Cal threshold. It failed to subtract Medicare Part B and other health insurance premiums under the senior-or-disabled non-MAGI income rules." -us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,"The model incorrectly treated Medicare eligibility as precluding Medicaid. Medicaid permits dual eligibility, and this head qualifies through California's optional senior-or-disabled pathway after satisfying its income, asset, and immigration tests." -us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model required an additional disability or low-income flag and overlooked that age 67 itself satisfies the aged-or-disabled categorical condition. It also failed to apply the pathway's deduction for Medicare Part B and other health insurance premiums from countable income. -us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the under-65 ACA adult pathway but then treated gross retirement income and minimal out-of-pocket medical expenses as the aged-pathway calculation. The relevant deduction is for health insurance premiums, including Medicare Part B, and it reduces countable income enough for eligibility without a spend-down." -us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The answer implies that gross retirement income was compared with a California Medicaid threshold. Under the senior-or-disabled non-MAGI pathway, Medicare Part B and other health insurance premiums are deducted, and the resulting countable income passes the test." -us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,The model treated the listed retirement income as exceeding the aged-pathway limit without applying the pathway's deductions. Deducting Medicare Part B and other health insurance premiums produces countable income within California's senior-or-disabled limit. -us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model applied an SSI-linked income level as the decisive Medicaid threshold. The optional senior-or-disabled category does not require SSI receipt and uses a non-MAGI income test that deducts Medicare Part B and other health insurance premiums. -us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"The model invented a rule excluding Medicare beneficiaries and full dual eligibles from Medicaid. Medicare eligibility does not bar Medicaid, and the head independently meets California's senior-or-disabled Medicaid conditions." -us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"The model incorrectly concluded that both $4,200 of bank assets and gross income fail the aged-pathway tests. The trace's applicable asset test is satisfied, and the income calculation deducts Medicare Part B and other health insurance premiums before applying the limit." -us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"The model applied a $2,000 Medi-Cal asset limit to the $4,200 bank balance. Under the applicable California optional senior-or-disabled pathway, the head passes the asset test; assets therefore do not defeat eligibility." -us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated age 67 as outside Medicaid unless disability, blindness, pregnancy, or SSI receipt was separately listed. Age 67 itself satisfies the aged-or-disabled condition for California's optional senior-or-disabled non-MAGI pathway, and SSI receipt is not required." -us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,The model correctly recognized that the ACA expansion group ends before age 67 but then compared listed retirement income with the aged Medi-Cal limit without the required deductions. Medicare Part B and other health insurance premiums reduce countable income enough to pass the non-MAGI test. -us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used the roughly $24,000 gross-income total as a shortcut and stopped without calculating non-MAGI countable income. The senior-or-disabled pathway deducts Medicare Part B and other health insurance premiums, yielding eligibility." -us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated Medicare and the end of ACA MAGI eligibility as exhausting the available pathways, then required disability, nursing-home status, or SSI. Age 67 independently opens California's optional senior-or-disabled non-MAGI pathway, which does not require SSI and deducts Medicare Part B and other health insurance premiums from countable income." -us,scenario_031,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated income below California's 200% broad-based categorical eligibility gross screen as sufficient for SNAP eligibility and then awarded the one-person minimum allotment. It failed to apply the remaining SNAP income eligibility calculation, under which this household's retirement, pension, and IRA income remains above the applicable limit after allowable deductions, so no minimum benefit is payable." -us,scenario_031,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model made the categorical-eligibility shortcut explicit: it used the 200% FPL CalFresh gross threshold to declare the household eligible and then paid the minimum allotment when the benefit formula went to zero. PolicyEngine's computation does not pay the minimum allotment to a household that fails the SNAP income eligibility test after deductions, and this household's countable income exceeds that limit." -us,scenario_031,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model characterized the household as having very low countable SNAP income despite $23,853 of annual Social Security retirement, taxable pension, and IRA income. After applying the available SNAP deductions, countable income is still above the applicable eligibility limit, so the positive $1,728 answer reflects an understated countable-income calculation." -us,scenario_031,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model correctly identified gross monthly income near $1,987 but stopped at the 200% categorical eligibility screen and then substituted a $24 monthly minimum allotment. It missed that the household's countable income after allowable deductions exceeds the SNAP income eligibility limit, and the minimum allotment is not available once the household is ineligible." -us,scenario_031,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model stopped at California tentative tax by applying the 1% bracket to its taxable-income estimate and explicitly applied no nonrefundable credits. It missed the California exemption credits for the age-67 filer, which exceed the roughly $75 tentative tax and reduce state income tax before refundable credits to $0." -us,scenario_032,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied only a broad CHIP age-and-income screen. It failed to determine that the child is Medicaid-eligible under Minnesota's OLDER_CHILD category, which automatically makes the child ineligible for CHIP." -us,scenario_032,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model combined Minnesota's Medicaid and CHIP expansion thresholds and treated falling below that combined ceiling as CHIP eligibility. At this income and age, the child qualifies for Medicaid under the OLDER_CHILD category, and that Medicaid eligibility excludes CHIP." -us,scenario_032,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model stopped after comparing household income with Minnesota's high child-coverage limit. It omitted the required Medicaid-first determination: the child qualifies under the OLDER_CHILD Medicaid category and therefore cannot qualify for CHIP. -us,scenario_032,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly treated Minnesota's 275% FPL child-coverage ceiling as a standalone CHIP threshold. The child's lower-income position places the child in the OLDER_CHILD Medicaid pathway, and existing Medicaid eligibility bars CHIP eligibility regardless of the lack of current health coverage." -us,scenario_032,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated the Medicaid/CHIP child-health coverage threshold with CHIP eligibility itself. The child qualifies specifically for Medicaid under the OLDER_CHILD category, so the Medicaid exclusion produces CHIP ineligibility." -us,scenario_032,child1_chip_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model performed only an income-below-275%-FPL comparison and labeled the result Minnesota CHIP. It skipped the Medicaid-first screen, under which the child qualifies as an OLDER_CHILD and is consequently excluded from CHIP." -us,scenario_032,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model acknowledged that CHIP depends on Medicaid not applying but then simply assumed CHIP eligibility without evaluating Medicaid. PolicyEngine finds the child Medicaid-eligible under the OLDER_CHILD category, so the stated condition for CHIP is not met." -us,scenario_032,child1_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated income below a combined CHIP/Medicaid coverage limit as sufficient for CHIP. The correct categorical sequence assigns the child to Medicaid's OLDER_CHILD pathway first, which makes CHIP unavailable." -us,scenario_032,child1_chip_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model used the CHIP income ceiling as the sole eligibility test. It omitted the prior Medicaid determination that places the child in the OLDER_CHILD category and therefore disqualifies the child from CHIP. -us,scenario_032,child1_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model reduced CHIP eligibility to being a child in a low-income Minnesota household. Low income instead places this child in the OLDER_CHILD Medicaid category, and Medicaid eligibility precludes CHIP." -us,scenario_032,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child1_chip_eligible, violating the required output contract." -us,scenario_032,child1_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model checked age, income, and Medicare status but failed to check Medicaid eligibility, the controlling exclusion for CHIP. The child qualifies for Medicaid under Minnesota's OLDER_CHILD category, so being under the cited CHIP ceiling and ineligible for Medicare does not establish CHIP eligibility." +us,scenario_030,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no TANF output, violating the required structured-output contract rather than making a substantive benefit calculation." +us,scenario_030,tanf,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly counted the unborn child as a current dependent child and constructed a two-person Texas TANF assistance unit. A pregnant adult with no born dependent child does not satisfy the family-with-dependent-children requirement, so the $271 monthly grant schedule does not apply." +us,scenario_031,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted a small positive taxable income without applying the full deduction sequence. The alimony adjustment reduces AGI to $13,243, and the single-filer standard deduction with the age-65 addition reduces taxable income and tax to zero." +us,scenario_031,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly denied the listed alimony deduction and used an estimated $10,207 post-TCJA-reversion standard deduction. PolicyEngine deducts the $1,165 alimony expense and applies the applicable 2026 single age-65-or-older standard deduction, leaving no taxable income." +us,scenario_031,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own calculation produced zero taxable income, zero tax, and zero nonrefundable credits, but it submitted $1,268.50. Its numeric output directly contradicts every computation step in its explanation." +us,scenario_031,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $5,357.50 of Social Security as taxable even though provisional income is below the $25,000 single-filer base, and it then failed to follow its own calculation that the elderly credit reduced tax to zero. The alimony adjustment yields $13,243 of AGI, the standard deduction eliminates taxable income, and the submitted $2,439 has no support in its stated arithmetic." +us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used a MAGI-style calculation and compared it with 138% FPL, even though the head qualifies through the non-MAGI senior-or-disabled category. It omitted the Medicare Part B and other health-insurance-premium deductions that reduce countable income enough to pass that category's income test." +us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model compared gross retirement income, less only alimony, with a 138% FPL threshold. The senior-or-disabled income test deducts health insurance premiums including Medicare Part B, and the resulting countable income satisfies the limit; the stated asset concern also conflicts with the satisfied asset test." +us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly described California's aged-and-disabled determination as MAGI-based and tested the full $23,853 against a 100% FPL figure. The applicable non-MAGI senior-or-disabled calculation deducts Medicare Part B and other health insurance premiums, producing income eligibility." +us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated the full $23,853 as countable income and compared it with a broad 100–138% FPL range. It omitted the non-MAGI senior-or-disabled pathway's health-insurance-premium deductions, including Medicare Part B, which make the head income-eligible." +us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model repeatedly routed the 67-year-old through MAGI expansion or SSI-linked thresholds and treated 138% FPL as controlling. PolicyEngine instead applies California's optional non-MAGI senior-or-disabled category, where Medicare Part B and other health premiums are deducted and the income test is met." +us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model substituted the SSI cash-benefit income standard for the optional senior-or-disabled Medicaid category's income test. SSI receipt is not required, and after the category-specific deduction of Medicare Part B and other health premiums, the head passes the applicable income test." +us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,The model compared gross monthly income with an estimated 138% FPL limit without applying the senior-or-disabled pathway's countable-income rules. Deducting Medicare Part B and other health insurance premiums brings countable income within the applicable limit. +us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model compared the full $23,853 with 138% FPL. The applicable non-MAGI senior-or-disabled test deducts Medicare Part B and other health insurance premiums before testing income, and that calculation yields eligibility." +us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies a gross-income comparison against a general California Medicaid limit. The correct senior-or-disabled derivation deducts Medicare Part B and other health insurance premiums from countable income, after which the head passes the income test." +us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated $23,853 as countable income under a 138% FPL test. California's non-MAGI senior-or-disabled pathway deducts Medicare Part B and other health premiums, and the reduced countable income satisfies its test." +us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Although it identified non-MAGI Medi-Cal, the model compared undeducted monthly income with 138% FPL. The senior-or-disabled calculation deducts Medicare Part B and other health insurance premiums, resulting in income eligibility." +us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated annual retirement income as exceeding the aged Medicaid limit without performing the category-specific countable-income calculation. Medicare Part B and other health insurance premiums are deductible, and the resulting income passes the senior-or-disabled test." +us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly treated Medicare eligibility as barring Medicaid eligibility. Dual Medicare-Medicaid eligibility is allowed, and this head qualifies through California's senior-or-disabled Medicaid category." +us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required a separately listed disability or other low-income flag and failed to recognize that age 67 itself satisfies the aged-or-disabled categorical condition. Under the senior-or-disabled pathway, premium-adjusted countable income and assets both pass." +us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,taxable_income_or_deductions,False,The model recognized that ACA adult Medicaid does not apply but concluded that retirement income exceeded the aged Medi-Cal standard after considering only the small listed medical expense. It omitted the deductible Medicare Part B and other health insurance premiums used by the senior-or-disabled income test. +us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,The answer implies that gross retirement income was compared directly with the California aged-pathway threshold. The required deduction of Medicare Part B and other health insurance premiums reduces countable income below that pathway's limit. +us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model treated annual retirement income as the amount tested under the aged pathway. The non-MAGI calculation deducts Medicare Part B and other health insurance premiums, yielding countable income within the limit." +us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model applied an SSI-linked income level instead of the optional senior-or-disabled Medicaid category's income test. That category does not require SSI receipt and deducts Medicare Part B and other health premiums, producing eligibility." +us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model invented an exclusion of Medicare-eligible people and full dual eligibles from Medicaid. Medicare eligibility does not preclude Medicaid, and the head independently meets California's senior-or-disabled Medicaid conditions." +us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"The model treated both the full $23,853 and $4,200 in bank assets as disqualifying. The engine's senior-or-disabled asset test is satisfied, while Medicare Part B and other health-premium deductions reduce countable income enough to satisfy the income test." +us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"The model applied a $2,000 asset ceiling and treated the $4,200 bank balance as disqualifying. The applicable California senior-or-disabled asset test is satisfied for this head, so the asserted resource bar does not apply." +us,scenario_031,head_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model applied a general 100% FPL limit to the full $23,853. The applicable optional senior-or-disabled pathway uses its own non-MAGI countable-income test and deducts Medicare Part B and other health premiums, resulting in eligibility." +us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated age 67 only as exclusion from the ACA expansion group and failed to recognize age 65 or older as satisfying the aged component of the senior-or-disabled category. Disability, blindness, and SSI receipt are not required when the applicant qualifies by age." +us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,The model correctly excluded the head from the ACA adult group but treated listed retirement income as exceeding the aged Medi-Cal limit. It omitted the senior-or-disabled pathway's deductions for Medicare Part B and other health insurance premiums. +us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model inferred that roughly $24,000 of gross income exceeded the senior limit without calculating non-MAGI countable income. Deducting Medicare Part B and other health insurance premiums under the senior-or-disabled pathway produces income eligibility." +us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated Medicare eligibility and absence of disability or SSI as defeating Medicaid eligibility, overlooking that age 67 independently establishes the aged categorical condition. It also used gross retirement income rather than countable income after Medicare Part B and other health-premium deductions." +us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model stated that the head lacked an eligibility category despite being age 67, which satisfies the aged condition in California's senior-or-disabled category. Under that pathway, deductible Medicare Part B and other health premiums reduce countable income enough to pass the income test." +us,scenario_031,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated income below California's 200% FPL broad-based categorical-eligibility threshold as guaranteeing SNAP eligibility and the $23 monthly minimum. The household fails the countable-net-income benefit determination, and the minimum allotment applies only after eligibility is established." +us,scenario_031,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly concluded that broad-based categorical eligibility guarantees the $23 monthly minimum even when 30% of net income exceeds the maximum allotment. That income calculation yields no eligibility for a positive allotment, so the minimum-benefit rule does not apply." +us,scenario_031,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that countable income was very low without applying the household's $23,853 annual retirement, IRA, and pension income to the SNAP net-income calculation. The resulting countable income produces no SNAP benefit, rather than the $144 monthly award implicit in its answer." +us,scenario_031,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated being under the 200% FPL categorical-eligibility threshold as sufficient to receive the 2026 minimum allotment of $24 per month. The household's countable net income fails the applicable SNAP eligibility and benefit determination, so no minimum allotment is payable." +us,scenario_031,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model explicitly applied no nonrefundable credits, omitting the California aged-or-blind exemption credit available to the 67-year-old filer. It also substituted an estimated $6,043 standard deduction for the applicable $5,706 deduction, but the decisive error is failing to reduce the resulting $75.37 tax by $312.93 of nonrefundable credits, which yields $0." +us,scenario_032,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied a CHIP income ceiling directly without first testing Medicaid eligibility. The child qualifies for Medicaid under the OLDER_CHILD category, which excludes the child from CHIP regardless of being below the broader child-coverage ceiling." +us,scenario_032,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model conflated Minnesota’s combined Medicaid/CHIP child-coverage threshold with direct CHIP eligibility. At this income and age, the child qualifies through the OLDER_CHILD Medicaid pathway, so CHIP eligibility is false." +us,scenario_032,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated income below Minnesota’s high child-health-coverage limit as sufficient for CHIP. It omitted the prior Medicaid determination: the child is Medicaid-eligible under OLDER_CHILD and therefore cannot qualify for CHIP. +us,scenario_032,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly labeled Minnesota’s coverage up to 275% FPL as CHIP eligibility across the entire income range. It failed to assign the child first to the OLDER_CHILD Medicaid category, whose eligibility precludes CHIP." +us,scenario_032,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model combined the Medicaid and CHIP income thresholds and concluded that eligibility for either program meant CHIP eligibility. The child specifically qualifies for Medicaid under OLDER_CHILD, and CHIP is unavailable to a Medicaid-eligible child." +us,scenario_032,child1_chip_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model compared income only with a 275% FPL ceiling and equated passing that test with Minnesota CHIP eligibility. It skipped the OLDER_CHILD Medicaid pathway, under which the child qualifies and is consequently excluded from CHIP." +us,scenario_032,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model acknowledged that CHIP depends on Medicaid not applying but then assumed CHIP eligibility without performing the Medicaid test. The child is Medicaid-eligible under OLDER_CHILD, so that stated condition fails and CHIP eligibility is false." +us,scenario_032,child1_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model conflated being below Minnesota’s combined CHIP/Medicaid child coverage limit with qualifying specifically for CHIP. The child falls into the OLDER_CHILD Medicaid category, and Medicaid eligibility bars CHIP eligibility." +us,scenario_032,child1_chip_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model used only the CHIP income ceiling and omitted Medicaid screening. The child qualifies for Medicaid under OLDER_CHILD, which makes the child ineligible for CHIP." +us,scenario_032,child1_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model reduced CHIP eligibility to child status and low household income. It failed to apply the OLDER_CHILD Medicaid pathway; because the child qualifies for Medicaid, CHIP eligibility is false." +us,scenario_032,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, violating the required output contract." +us,scenario_032,child1_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated age, income below an asserted CHIP threshold, and lack of Medicare eligibility as sufficient conditions for CHIP. It omitted the controlling Medicaid screen: the child qualifies under the OLDER_CHILD Medicaid category and is therefore ineligible for CHIP." us,scenario_032,child1_early_head_start_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_032,child1_head_start_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly recognized that counted household income exceeds the 100% FPL income test, but then converted Head Start's limited over-income enrollment discretion into individual eligibility. PolicyEngine's binary eligibility output does not award eligibility from the 10% over-income enrollment allowance, and the household has no categorical eligibility pathway." -us,scenario_032,child1_head_start_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model treated the household's income as generally low enough without applying the actual Head Start poverty comparison. PolicyEngine uses adjusted gross income of $30,916 against the $27,320 federal poverty guideline for the tax unit, so the income test fails, and no categorical pathway overrides it." -us,scenario_032,child1_head_start_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model misstated the poverty comparison by saying the household's countable income is below the guideline for a family of three. The correct computation compares $30,916 of adjusted gross income to a $27,320 poverty guideline, placing the household above the Head Start income eligibility threshold." -us,scenario_032,child1_head_start_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable value or supporting explanation for this requested output. The correct derivation yields not eligible because adjusted gross income of $30,916 exceeds the $27,320 poverty guideline and no categorical eligibility pathway applies." -us,scenario_032,child1_head_start_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model relied on age-range flexibility and possible mixed-age program participation instead of applying the income and categorical eligibility tests required by the benchmark output. Age alone does not establish Head Start eligibility, and this household fails the income test with $30,916 above the $27,320 poverty guideline and has no categorical pathway." -us,scenario_032,child1_head_start_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used the wrong poverty guideline and an inflated Head Start income threshold of 130-150% FPL. PolicyEngine applies the Head Start income test at the poverty guideline, comparing adjusted gross income of $30,916 to $27,320, so the household is over income and has no categorical eligibility route." -us,scenario_032,child1_medicaid_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied the 138% FPL adult Medicaid expansion cutoff to a six-year-old instead of Minnesota's OLDER_CHILD Medicaid income limit. At 1.56 times FPL, the dependent child qualifies for Medicaid rather than being shifted to CHIP." -us,scenario_032,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicit eligibility indicator as grounds for denial even though eligibility was the requested derived output. It also relied on the $29,000 wage amount instead of comparing the resulting 1.56-times-FPL MAGI level with Minnesota's OLDER_CHILD Medicaid threshold, which yields eligibility." -us,scenario_032,child1_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model placed the child in Minnesota's separate CHIP income range without first applying the OLDER_CHILD Medicaid pathway. The age-6 dependent's MAGI of 1.56 times FPL is within the applicable Medicaid limit, so the CHIP-only classification is wrong." -us,scenario_032,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the requested output entirely. The required derivation classifies the age-6 dependent as OLDER_CHILD and compares 1.56-times-FPL MAGI with Minnesota's applicable Medicaid limit, yielding value 1." -us,scenario_032,child1_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable value for child1_medicare_eligible, so it failed the required output contract rather than applying a Medicare eligibility rule." -us,scenario_032,child1_medicare_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model incorrectly treated the head's Social Security dependent benefits as evidence that the child inherits or qualifies for Medicare through a household disability pathway. Medicare eligibility is person-specific here; the 6-year-old child is under 65 and has no listed ESRD, ALS, or other qualifying individual Medicare condition, so the correct value is 0." -us,scenario_032,child1_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated a 6-year-old as within WIC's child age category by inventing an extension to age 6 or school start. WIC child categorical eligibility ends before age 5, so the income comparison to 185% of poverty is irrelevant for this child." -us,scenario_032,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model's entire answer rests on the false premise that age 6 is a WIC-eligible child age group. PolicyEngine's WIC child pathway requires the child to be under age 5, so Child 1 fails categorical eligibility before any income test matters." -us,scenario_032,child1_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model noticed the under-5 issue but overrode it with an income-based shortcut, treating borderline income qualification as enough for WIC. WIC requires both categorical eligibility and income eligibility, and a 6-year-old child fails the categorical age test." -us,scenario_032,child1_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable child1_wic_eligible value or supporting explanation. This is a missing-output failure rather than a substantive tax-benefit calculation. -us,scenario_032,child1_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,The model incorrectly placed Child 1 in the WIC child category despite the child being age 6. The WIC income limit for a household of three does not determine eligibility when the child fails the under-age-5 categorical requirement. -us,scenario_032,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model initially reached the correct zero result, then replaced PolicyEngine's 2026 parameters with a post-TCJA-sunset shortcut: a much smaller MFJ standard deduction and a $1,000 CTC. That produced taxable income of $14,766, tax of $1,477, and a residual $477 after CTC, instead of applying the reference's 2026 nonrefundable CTC amount that fully eliminates the pre-refundable tax liability." -us,scenario_032,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model did not provide a parseable value or explanation for federal_income_tax_before_refundable_credits. It failed the output contract rather than performing the required computation that subtracts the nonrefundable CTC from income tax before refundable credits. -us,scenario_032,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable value or explanation for federal_income_tax_before_refundable_credits. It failed the output contract rather than performing the required computation that subtracts the nonrefundable CTC from income tax before refundable credits. -us,scenario_032,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model rounded the $4,427 EITC down to $4,420 despite identifying the applicable maximum and correctly computing the $1,700 refundable CTC. That unsupported rounding caused the entire $7 shortfall." -us,scenario_032,federal_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model incorrectly treated the spouse and child as three EITC qualifying children and then applied an unexplained phaseout. A spouse is not a qualifying child; this return has one qualifying child and receives a $4,427 EITC plus a $1,700 refundable CTC." -us,scenario_032,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model reduced the EITC to $4,328 and then added an unexplained $215 “estimate refinement.” The trace applies the 2026 one-child MFJ parameters to produce a $4,427 EITC, with only the separate $1,700 refundable CTC added." -us,scenario_032,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model incorrectly placed $29,000 of earnings in the EITC phaseout range and reduced the credit to $3,372. The applicable 2026 computation yields the full $4,427 EITC, which combines with the correctly identified $1,700 refundable CTC." -us,scenario_032,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly reduced the one-child EITC to about $3,412 and then inserted an unexplained $1,100 amount. The only refundable components are the $4,427 EITC and the $1,700 refundable CTC." -us,scenario_032,federal_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated part of the Social Security dependent benefits as taxable and then made the refundable CTC equal to the unused portion of a $2,000 CTC. The computation instead uses a $4,427 EITC and the statutory $1,700 refundable CTC cap, without increasing that refundable cap based on estimated pre-credit tax." -us,scenario_032,federal_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented an additional refundable credit for the 18-year-old spouse after correctly stating that the spouse was not another qualifying child. No spouse-dependent refundable credit applies; the total contains only the $4,427 EITC and $1,700 refundable CTC." -us,scenario_032,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 refundable CTC cap and a $3,000 earnings threshold instead of the applicable $1,700 cap and $2,500 threshold. It also overstated the EITC as $4,488 rather than $4,427." -us,scenario_032,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the wrong EITC schedule to produce $3,599.34 and capped the refundable CTC at an obsolete $1,000. The applicable amounts are $4,427 and $1,700." -us,scenario_032,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its total is inconsistent with the applicable $4,427 EITC plus $1,700 refundable CTC. The answer implies that it understated one or both components by $665 instead of applying the 2026 one-child credit parameters." -us,scenario_032,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a pre-TCJA $1,000 Additional Child Tax Credit cap instead of the applicable $1,700 refundable cap. It also overstated the EITC by $13; the correct components are $4,427 and $1,700." -us,scenario_032,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model understated the EITC as $4,014 and used an obsolete $1,000 refundable CTC cap. Applying the 2026 rules yields $4,427 of EITC and $1,700 of refundable CTC." -us,scenario_032,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly applied post-expiration rules with a $1,000 CTC cap and a $3,000 earnings threshold. The applicable refundable CTC is $1,700, and the EITC is $4,427 rather than $4,428." -us,scenario_032,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no output for federal_refundable_credits. It therefore omitted the $4,427 EITC and $1,700 refundable CTC required by the requested output contract." -us,scenario_032,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to recognize both refundable-credit pathways. A married couple with $29,000 of earnings and one qualifying child receives a $4,427 EITC and a $1,700 refundable CTC." -us,scenario_032,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of explicitly supplied credit amounts as absence of eligibility, even though the benchmark asks it to calculate credits from wages and family facts. Those facts produce a $4,427 EITC and a $1,700 refundable CTC." -us,scenario_032,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an $1,800 refundable CTC limit instead of the applicable $1,700 limit. Its EITC amount of $4,427 was correct, so the cap error overstated the total by exactly $100." -us,scenario_032,federal_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The answer is $80 below the derivation and gives no component amounts. The correct computation is exactly $4,427 of EITC plus $1,700 of refundable CTC, rather than an unspecified approximation of those credits." -us,scenario_032,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model named EITC and CTC but did not apply either credit's 2026 parameters. Their computed refundable amounts are $4,427 and $1,700, not a combined $2,500 estimate." -us,scenario_032,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 refundable CTC cap and overstated the EITC as $4,436. The applicable components are a $4,427 EITC and a $1,700 refundable CTC." -us,scenario_032,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 refundable CTC cap and overstated the EITC as $4,439. Its invented $30,916 AGI does not alter the traced result of $4,427 plus $1,700." -us,scenario_032,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output for federal_refundable_credits. It therefore omitted the $4,427 EITC and $1,700 refundable CTC required by the requested output contract." -us,scenario_032,federal_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model incorrectly used the two-child EITC schedule even though the household has only one EITC qualifying child, then omitted the refundable CTC from its submitted total despite mentioning it. The correct total uses the one-child $4,427 EITC plus a $1,700 refundable CTC." -us,scenario_032,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model repeatedly substituted incorrect EITC maxima and phaseout thresholds, then arbitrarily reduced the refundable CTC to $800 despite its own phase-in calculation exceeding the cap. The applicable calculation yields a $4,427 EITC and the full $1,700 refundable CTC." -us,scenario_032,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model treated the federal 130% FPL test and SNAP/TANF direct certification as the only free-meal pathways. It omitted Minnesota's universal free school meals program, which covers the enrolled K-12 child despite household income at 1.56 times the poverty guideline." -us,scenario_032,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,The model applied only the National School Lunch Program's federal income threshold and concluded that income above 130% FPL precluded free meals. Minnesota's universal program independently assigns the enrolled K-12 child to the FREE tier. -us,scenario_032,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,The model used income above 130% FPL as dispositive and omitted Minnesota's universal free school meals pathway. The enrolled K-12 child receives free meals under that state program without satisfying the federal income test. -us,scenario_032,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model stopped after comparing $42,664 with the federal 130% FPL threshold. It failed to apply Minnesota's universal free school meals program, which makes the school-age child eligible regardless of that comparison." -us,scenario_032,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,The model incorrectly required either stated free-meal eligibility or categorical eligibility through SNAP/TANF. Minnesota's universal free school meals program is a separate qualifying pathway and produces positive support for the enrolled K-12 child. -us,scenario_032,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model treated income below the federal limit or categorical status as necessary for free meals. It omitted Minnesota's universal program, under which the enrolled K-12 child qualifies even at 1.56 times the poverty guideline." -us,scenario_032,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,The model assumed income above the federal free-meal threshold prevents positive support. Minnesota's universal free school meals rule overrides that income-based exclusion for the enrolled K-12 child. -us,scenario_032,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model limited eligibility to the federal income test and direct certification. It omitted Minnesota's universal free school meals program, the pathway that assigns the child to the FREE tier." -us,scenario_032,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,The model correctly identified income at about 156% of the poverty guideline but incorrectly treated the 130% federal limit as controlling. Minnesota's universal free school meals program makes the enrolled K-12 child eligible above that limit. -us,scenario_032,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model performed only the federal 130% FPL comparison and used the result to deny eligibility. It failed to apply Minnesota's universal free school meals program, which supplies the decisive eligibility pathway." -us,scenario_032,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer for the requested output. The required derivation applies Minnesota's universal free school meals program to the enrolled K-12 child and yields Yes. -us,scenario_032,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model treated PolicyEngine's federal income test as the sole determinant and denied eligibility at 156% FPL. It omitted Minnesota's universal free school meals pathway, which yields positive free-meal support." +us,scenario_032,child1_head_start_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model correctly found income above 100% FPL but converted Head Start's limited allowance to enroll some over-income children into an individual entitlement. It also asserted categorical eligibility without identifying any qualifying categorical circumstance in the household. +us,scenario_032,child1_head_start_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model declared categorical and income eligibility without naming a categorical pathway or applying the traced income test. Adjusted gross income is $30,916, above the $27,320 federal poverty guideline, and no categorical pathway applies." +us,scenario_032,child1_head_start_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model reversed the income comparison by claiming countable income was below the poverty guideline. The applicable figures are $30,916 of adjusted gross income and a $27,320 guideline, so the income test fails." +us,scenario_032,child1_head_start_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the required output entirely, so it did not submit the binary Head Start eligibility value or a supporting explanation." +us,scenario_032,child1_head_start_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated possible participation by an older child in a mixed-age or enrichment program as establishing Head Start eligibility. It never applied either required eligibility route: no categorical pathway exists, and $30,916 of adjusted gross income exceeds the $27,320 guideline." +us,scenario_032,child1_head_start_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used $29,000 of wages instead of the traced $30,916 adjusted gross income, used an incorrect family-of-three poverty guideline, and treated 130–150% FPL as the ordinary Head Start income cutoff. The applicable test compares $30,916 with $27,320, and the household fails it." +us,scenario_032,child1_head_start_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted that income was within the Head Start limit without performing the required comparison. Adjusted gross income of $30,916 exceeds the $27,320 federal poverty guideline, and no categorical eligibility pathway applies." +us,scenario_032,child1_medicaid_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied the 138% FPL adult Medicaid expansion cutoff to a six-year-old and then assigned the child to CHIP. Minnesota instead tests the child under the OLDER_CHILD Medicaid category, whose income limit includes MAGI at 1.56 times FPL." +us,scenario_032,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicit eligibility indicator as grounds for denial instead of deriving eligibility from the supplied age, state, year, and household income. Minnesota's OLDER_CHILD Medicaid test covers this six-year-old at 1.56 times FPL." +us,scenario_032,child1_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model incorrectly placed the child in a separate CHIP income band rather than applying Minnesota's OLDER_CHILD Medicaid threshold first. The child's MAGI of 1.56 times FPL satisfies the Medicaid limit for that age category. +us,scenario_032,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model omitted the required child1_medicaid_eligible output and supplied no explanation. The required derivation classifies the six-year-old as OLDER_CHILD and returns eligible at 1.56 times FPL. +us,scenario_032,child1_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the required child1_medicare_eligible output, so it never applied the child's age and medical-status facts to Medicare eligibility." +us,scenario_032,child1_medicare_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly transferred a supposed disability-based Medicare pathway from the head's Social Security dependent benefits to the child. Social Security dependent benefits do not confer Medicare on dependents, and this six-year-old has no listed ESRD, ALS, or individual disability-based Medicare entitlement." +us,scenario_032,child1_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly extended WIC child eligibility to age six or the start of school. Federal WIC eligibility for children ends at the fifth birthday, so its income-limit analysis cannot make this six-year-old eligible." +us,scenario_032,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model directly classified age six as a WIC-eligible child age group. WIC child eligibility ends at the fifth birthday, making Child 1 categorically ineligible." +us,scenario_032,child1_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model noticed that the child was six but nevertheless treated income qualification as sufficient. A six-year-old fails WIC’s categorical age requirement, and income qualification does not override that failure." +us,scenario_032,child1_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_wic_eligible, violating the required output contract." +us,scenario_032,child1_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly placed a six-year-old in the WIC-eligible child category and then relied on the 185% income screen. WIC child eligibility ends at the fifth birthday, so the income comparison is irrelevant here." +us,scenario_032,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated age six as satisfying WIC’s child category and used income eligibility to conclude yes. Because WIC coverage for children ends at the fifth birthday, Child 1 fails the categorical requirement before the income test can establish eligibility." +us,scenario_032,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly assumed that the TCJA rules expired for 2026, substituted a roughly $16,150 MFJ standard deduction and a $1,000 CTC, and thereby produced $1,477 of pre-credit tax minus $1,000. Under the applicable 2026 parameters, the $2,200 CTC supplies $500 of nonrefundable credit, which fully reduces the liability to $0; its remaining $1,700 and the EITC are refundable and do not enter this output." +us,scenario_032,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output. The required computation applies $500 of nonrefundable CTC to eliminate the positive pre-credit liability, yielding $0 before refundable credits." +us,scenario_032,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output. The required computation applies $500 of nonrefundable CTC to eliminate the positive pre-credit liability, yielding $0 before refundable credits." +us,scenario_032,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model rounded the $4,427 EITC down to approximately $4,420 instead of applying the exact 2026 amount. Its $1,700 refundable CTC was correct, leaving the entire $7 error in the EITC component." +us,scenario_032,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated the spouse and household structure as producing three EITC qualifying children; a spouse is never the taxpayer's qualifying child, and only the age-6 child qualifies. It also used a $1,600 refundable CTC instead of $1,700 and then applied an unsupported phaseout." +us,scenario_032,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model first understated the EITC as $4,328 and then added an unexplained $215 “estimate refinement” that corresponds to no refundable federal credit. The only applicable components are the $4,427 EITC and $1,700 refundable CTC." +us,scenario_032,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model incorrectly phased the one-child MFJ EITC down to $3,372 at $29,000 of earned income. The applicable EITC is $4,427, which combines with the correctly calculated $1,700 refundable CTC." +us,scenario_032,federal_refundable_credits,claude-opus-5,llm_error,other,False,"The model reduced the EITC to roughly $3,412 and then inserted an unexplained $1,100 refundable amount beyond the stated $1,700 child credit. No such additional refundable credit applies; the total consists solely of a $4,427 EITC and $1,700 refundable CTC." +us,scenario_032,federal_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used the wrong Social Security taxation thresholds by applying the $34,000 threshold associated with a non-joint filer instead of the MFJ computation, creating taxable Social Security and pre-credit income tax. It then improperly made the refundable CTC equal to the unused portion of a $2,000 CTC rather than applying the $1,700 refundable cap; the correct refundable components are $4,427 and $1,700." +us,scenario_032,federal_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"After correctly identifying one qualifying child and approximately $6,000 of applicable credits, the model invented an “additional refundable credit for spouse as a qualifying dependent.” A spouse does not generate a refundable dependent credit, so only the $4,427 EITC and $1,700 refundable CTC apply." +us,scenario_032,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 ACTC cap and $3,000 earned-income threshold instead of the applicable $1,700 refundable CTC cap and $2,500 threshold. It also overstated the EITC by $61; the correct components are $4,427 and $1,700." +us,scenario_032,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 refundable CTC cap instead of $1,700 and understated the EITC as $3,599.34 instead of $4,427. Both component parameters were wrong." +us,scenario_032,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $5,462 total is $665 below the $6,127 obtained from the applicable $4,427 EITC and $1,700 refundable CTC. The answer therefore failed to apply the exact 2026 amounts for the two credits it identified." +us,scenario_032,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model reverted the refundable CTC to the pre-TCJA $1,000 amount instead of applying the $1,700 cap. It also overstated the EITC by $13; the applicable EITC is $4,427." +us,scenario_032,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 ACTC amount instead of $1,700 and understated the EITC as $4,014 rather than $4,427. Those two parameter errors account for the shortfall." +us,scenario_032,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly assumed expiration-law parameters, replacing the applicable $1,700 refundable CTC with a $1,000 cap and a $3,000 threshold. The EITC is $4,427, not $4,428, so the correct total is $6,127." +us,scenario_032,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for federal_refundable_credits. +us,scenario_032,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to recognize that $29,000 of wages and one qualifying child generate both the EITC and refundable CTC. Those credits are $4,427 and $1,700 respectively, not zero." +us,scenario_032,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of explicitly supplied credit amounts as absence of eligibility, even though the task requires deriving credits from wages, filing status, and the qualifying child. Those facts produce a $4,427 EITC and $1,700 refundable CTC." +us,scenario_032,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an $1,800 refundable CTC limit instead of the applicable $1,700 limit. Its $4,427 EITC was correct, so the CTC cap alone caused the $100 overstatement." +us,scenario_032,federal_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the correct two credits but understated their combined value by $80. Applying the exact $4,427 EITC and $1,700 refundable CTC yields $6,127." +us,scenario_032,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model named EITC and CTC but supplied a combined $2,500 without calculating either component. The applicable amounts are $4,427 for EITC and $1,700 for the refundable CTC." +us,scenario_032,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used the obsolete $1,000 refundable CTC cap instead of $1,700 and overstated the EITC by $9. The correct component amounts are $4,427 and $1,700." +us,scenario_032,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model capped the refundable CTC at $1,000 instead of $1,700 and overstated the EITC as $4,439 instead of $4,427. Its invented taxable AGI of $30,916 was also inconsistent with the prompt's zero-default treatment of unlisted income." +us,scenario_032,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly used the $1,700 refundable CTC cap but overstated the EITC as $4,449. The exact EITC is $4,427, producing a total of $6,127." +us,scenario_032,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for federal_refundable_credits. +us,scenario_032,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly classified the household as having two EITC qualifying children when only the age-6 child qualifies, and it then omitted the refundable CTC from its submitted total despite mentioning it. The correct total includes a $4,427 one-child EITC plus a $1,700 refundable CTC." +us,scenario_032,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model cycled through invented EITC maxima, phaseout thresholds, and refundable CTC caps, then submitted a final $4,509 EITC and $800 ACTC unsupported by its own calculations. The applicable fixed components are a $4,427 EITC and $1,700 refundable CTC." +us,scenario_032,federal_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model drastically understated both components, assigning only $1,171 of EITC and $937 of refundable CTC. The household's wages and one qualifying child produce $4,427 and $1,700 respectively." +us,scenario_032,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model treated the federal 130% FPL test and SNAP/TANF direct certification as the only qualifying pathways. It omitted Minnesota's universal free school meals program, which makes the K-12 child eligible at the household's 1.56 FPL ratio." +us,scenario_032,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model applied the National School Lunch Program's 130% FPL income threshold as dispositive after calculating $42,664 of household income. It failed to apply Minnesota's universal free school meals program, which extends free meals beyond that federal income threshold." +us,scenario_032,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model denied eligibility solely because $42,664 exceeded its 130% FPL threshold. It omitted Minnesota's universal free school meals program, under which the enrolled K-12 child qualifies regardless of this income-test result." +us,scenario_032,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model correctly totaled wages and Social Security benefits to $42,664 but incorrectly stopped at the federal 130% FPL test. Minnesota's universal free school meals program supplies the qualifying pathway despite the household's 1.56 FPL ratio." +us,scenario_032,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed that absent SNAP/TANF receipt or an explicitly listed free-meal status, no free-meal pathway existed. It failed to derive eligibility from the listed Minnesota residence and Minnesota's universal free school meals program." +us,scenario_032,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model treated income below the federal limit or categorical status as necessary for free meals. It omitted Minnesota's universal program, which makes the household's K-12 child eligible even though income is 1.56 times the poverty guideline and categorical eligibility is false." +us,scenario_032,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,The model concluded that income above the federal free-meal threshold prevented positive support. Minnesota's universal free school meals rule overrides that income-based denial for the enrolled K-12 child. +us,scenario_032,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model limited eligibility to the federal income threshold and direct certification. It omitted Minnesota's universal free school meals pathway, which produces positive annual free-meal support without either condition." +us,scenario_032,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,The model correctly recognized that household income was about 156% of the poverty guideline but incorrectly treated the 130% federal limit as controlling. Minnesota's universal free school meals program makes the K-12 child eligible above that limit. +us,scenario_032,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model used the household's roughly 156.7% FPL ratio to deny eligibility under the 130% federal threshold. It failed to apply Minnesota's universal free school meals program, the decisive state pathway." +us,scenario_032,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,"The model denied free meals solely because household income exceeded 130% FPL. It omitted Minnesota's universal free school meals program, which extends free eligibility to the enrolled child regardless of that threshold result." +us,scenario_032,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer or explanation for the requested output. It therefore failed the required structured-output contract rather than performing the Minnesota universal free school meals eligibility determination. +us,scenario_032,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model applied only PolicyEngine's federal 130% FPL income test and denied eligibility at about 156% FPL. It omitted Minnesota's universal free school meals program, which independently sets the household's school-meal tier to FREE." us,scenario_032,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_032,head_medicaid_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model excluded the Social Security dependent benefits and consequently placed the head below the 138% FPL adult limit. The applicable MAGI computation produces 1.56 times FPL, and the head has no other Medicaid eligibility category." -us,scenario_032,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model invented a higher Minnesota parent/caretaker Medicaid threshold after acknowledging that income including Social Security exceeded the expansion limit. The head qualifies through neither the applicable MAGI adult limit at 1.56 times FPL nor any parent/caretaker or other categorical pathway. -us,scenario_032,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model excluded the Social Security dependent benefits from household MAGI and therefore understated the income level. The correct MAGI result is 1.56 times FPL, above the applicable adult limit, with medicaid_category NONE." -us,scenario_032,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model invented a 275% FPL Minnesota Medicaid threshold for parents or caretaker relatives and used it to override the applicable adult limit. At 1.56 times FPL, the head exceeds the relevant MAGI threshold and qualifies through no caretaker or other category." -us,scenario_032,head_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly treated an income item—the Social Security dependent benefit—as a fourth household member. The household has three people, and the proper Medicaid MAGI calculation places the head at 1.56 times FPL, above the applicable limit." -us,scenario_032,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The answer applied a generic low-income shortcut without testing the head against Minnesota’s Medicaid categories. The head is at 1.56 times FPL and the categorical determination is NONE, so no Medicaid pathway applies." -us,scenario_032,head_medicaid_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model asserted that household MAGI fell within the adult expansion range, but the Medicaid MAGI calculation yields 1.56 times FPL. That exceeds the applicable limit, and the head has no alternative eligibility category." -us,scenario_032,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,The model treated general low income as sufficient for Medicaid and omitted the applicable income and category tests. The head’s MAGI is 1.56 times FPL and medicaid_category is NONE. -us,scenario_032,head_medicaid_eligible,grok-4.5,llm_error,categorical_eligibility,False,"The model calculated MAGI as $30,916 and placed it at 113% FPL, omitting income included in the Medicaid MAGI computation. The correct result is 1.56 times FPL, above Minnesota’s applicable adult limit, with no alternative category." -us,scenario_032,head_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model understated the head’s Medicaid income level as 113.6% FPL and applied the 138% expansion limit to that figure. The applicable MAGI calculation yields 1.56 times FPL, and no other Medicaid category applies." -us,scenario_032,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for head_medicaid_eligible, so the required output was missing." -us,scenario_032,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the household as being at the Medicaid expansion cutoff, but the head’s MAGI income level is 1.56 times FPL. This exceeds the applicable adult limit, and absence of Medicare eligibility does not create a Medicaid pathway." -us,scenario_032,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model discarded the Social Security dependent benefits from MAGI and tested only $29,000 of wages against the expansion threshold. The applicable Medicaid MAGI calculation places the head at 1.56 times FPL, above the limit, and no alternative category applies." +us,scenario_032,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model excluded the Social Security dependent benefits and tested only $29,000 of wages against 138% FPL. The Medicaid MAGI calculation yields 1.56 times FPL, so the head is above the adult MAGI limit and has no qualifying category." +us,scenario_032,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model acknowledged income near $42,664 but invented a higher Minnesota Medicaid threshold for parents or caretakers. The head’s MAGI is 1.56 times FPL, and the presence of a child does not place the head in a qualifying Medicaid category at that income." +us,scenario_032,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded the Social Security dependent benefits and treated wages alone as Medicaid MAGI. The engine’s MAGI calculation is 1.56 times FPL, above the expansion-adult limit, leaving the head with category NONE." +us,scenario_032,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model invented a 275% FPL Medicaid threshold for Minnesota parents or caretaker relatives and used it after recognizing that income exceeded 138% FPL. No such eligibility pathway applies here: at 1.56 times FPL, the head qualifies through none of Minnesota’s Medicaid categories." +us,scenario_032,head_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted an income item as a fourth household member, inflating household size from three to four and therefore inflating the poverty threshold. With the correct Medicaid household computation, MAGI is 1.56 times FPL and exceeds the adult limit." +us,scenario_032,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model applied a generic low-income shortcut without testing the head against Minnesota’s specific Medicaid pathways. The correct comparison places MAGI at 1.56 times FPL, above the applicable adult limit, with no alternate qualifying category." +us,scenario_032,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that household MAGI fell within Minnesota’s expansion range, but the Medicaid MAGI calculation equals 1.56 times FPL. That exceeds the applicable adult threshold, and the head has no other eligibility category." +us,scenario_032,head_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model substituted a generic low-income characterization for the required income-threshold and categorical tests. MAGI is 1.56 times FPL, above the adult limit, and no other Medicaid pathway applies." +us,scenario_032,head_medicaid_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model calculated MAGI as $30,916 and 113% FPL, understating the income measure used by the engine. The correct Medicaid MAGI is 1.56 times FPL, which exceeds the 138% expansion threshold." +us,scenario_032,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated the head’s Medicaid income level as about 113.6% FPL. The engine’s MAGI calculation is 1.56 times FPL, above the adult expansion limit, and the head has no alternate qualifying category." +us,scenario_032,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required output contract rather than completing the Medicaid eligibility calculation. +us,scenario_032,head_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model treated the household as being at the 138% expansion boundary. The correct Medicaid MAGI level is 1.56 times FPL, above that boundary, and being under age 65 and ineligible for Medicare does not create another Medicaid pathway." +us,scenario_032,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model discarded the Social Security dependent benefits from the head’s Medicaid MAGI and compared wages alone with 138% FPL. The correct MAGI computation produces 1.56 times FPL, so the head exceeds the adult limit and qualifies through no other category." us,scenario_032,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_032,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated WIC as a household-context benefit for the head once it believed the household had a categorically eligible member, instead of applying WIC categorical eligibility to the head herself. The head is age 50 and has no listed pregnancy, postpartum, or breastfeeding status, so she fails the person-level WIC category regardless of the child in the household. It also used only $29,000 of wages for the income screen and omitted $13,664 of Social Security dependent benefits, understating household income." -us,scenario_032,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly asserted that Minnesota has no mandatory employee state payroll tax in 2026. It omitted the $127.60 Minnesota paid-leave contribution on $29,000 of taxable wages." -us,scenario_032,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model omitted Minnesota's $127.60 employee paid-leave contribution and then submitted $2,175.90 despite repeatedly deriving federal FICA of $2,218.50. Adding the state contribution to FICA yields $2,346.10." -us,scenario_032,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,The model incorrectly limited payroll tax to 7.65% federal FICA by asserting that Minnesota has no mandatory employee state payroll tax. It omitted the $127.60 Minnesota paid-leave contribution. -us,scenario_032,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model correctly computed $2,218.50 of federal FICA but incorrectly stated that Minnesota has no mandatory employee payroll tax. The missing Minnesota paid-leave contribution is $127.60." -us,scenario_032,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"The model incorrectly excluded mandatory Minnesota employee payroll tax. Its $2,218.50 includes only Social Security and Medicare and omits the $127.60 paid-leave contribution." -us,scenario_032,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly asserted that Minnesota imposes no separate employee-side payroll tax. For 2026, the payroll-tax output includes a $127.60 Minnesota paid-leave contribution on the head's wages." -us,scenario_032,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly stated that no Minnesota paid-family-leave employee contribution applies in 2026. It therefore omitted the $127.60 state payroll tax from its otherwise correct $2,218.50 FICA calculation." -us,scenario_032,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"The model included Minnesota paid leave but applied a 0.35% employee share, producing $101.50. The applicable contribution in this computation is $127.60 on $29,000 of Minnesota paid-leave taxable wages, so total payroll tax is $2,346.10." -us,scenario_032,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,The model treated federal FICA at 7.65% as the entire payroll-tax output. It omitted Minnesota's $127.60 mandatory employee paid-leave contribution. -us,scenario_032,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model computed and rounded federal Social Security and Medicare taxes while omitting the mandatory $127.60 Minnesota paid-leave contribution. The output requires federal and state employee payroll taxes together. -us,scenario_032,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,The model stopped after applying the 7.65% federal FICA rate to wages. It failed to add the $127.60 Minnesota employee paid-leave contribution. -us,scenario_032,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"The model equated employee payroll tax with federal FICA alone. Minnesota's mandatory paid-leave contribution adds $127.60 to the $2,218.50 federal amount." -us,scenario_032,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,The model applied only the 7.65% federal FICA rate. It omitted the $127.60 Minnesota paid-leave employee contribution included in the requested payroll-tax output. -us,scenario_032,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, violating the required structured-output contract." -us,scenario_032,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model's stated components produce $2,218.50, not its submitted $4,428.60, and its assertion that Minnesota has no mandatory employee payroll tax omits the $127.60 paid-leave contribution. The three required components total $2,346.10." -us,scenario_032,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"The model correctly derived $2,218.50 of federal FICA, then discarded that result and invented an unsupported 5.24% rate based on a nonexistent Social Security wage-base interaction. It also omitted Minnesota's $127.60 paid-leave contribution, leaving the submitted $1,520 disconnected from the required computation." -us,scenario_032,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,The model incorrectly stated that no state payroll tax applies and reported federal FICA alone. It omitted Minnesota's $127.60 employee paid-leave contribution. -us,scenario_032,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The model rounded federal FICA to $2,219 and treated it as the complete payroll tax. It omitted the $127.60 Minnesota paid-leave contribution." -us,scenario_032,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model included only employee Social Security and Medicare taxes. It omitted the mandatory $127.60 Minnesota employee paid-leave contribution. -us,scenario_032,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that no mandatory Minnesota employee payroll tax applies. The missing Minnesota paid-leave contribution is $127.60 on the head's $29,000 of taxable wages." -us,scenario_032,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"The model stopped after calculating $2,218.50 of federal FICA. It failed to include the $127.60 Minnesota paid-leave employee contribution required by the payroll-tax output." -us,scenario_032,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, violating the required structured-output contract." -us,scenario_032,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model rounded $2,218.50 of federal FICA to $2,219 but omitted Minnesota's $127.60 employee paid-leave contribution. Rounding does not account for the missing state component." -us,scenario_032,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly concluded that Minnesota has no mandatory employee state payroll tax and rounded federal FICA alone. Minnesota's 2026 paid-leave contribution adds $127.60, producing $2,346.10 rather than $2,219." -us,scenario_032,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"The model applied the federal 185% poverty limit but omitted Minnesota's universal free school meals program. Universal free eligibility assigns the superior FREE tier, leaving no positive reduced-price support." -us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"The model classified the household solely from the federal 130%–185% income band and concluded that income above the federal free-meal threshold prevents free meals. Minnesota provides universal free meals regardless of income, and that FREE classification supersedes reduced-price eligibility." -us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"The model used the ordinary federal income bands while omitting Minnesota's universal free school meals rule. The child receives the FREE tier under the state program, so PolicyEngine returns no reduced-price support." -us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model treated the household's poverty ratio as sufficient to assign the reduced-price tier. Minnesota's universal program instead assigns the FREE tier, which supersedes reduced-price eligibility." -us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model repeatedly applied the federal 130%–185% band and explicitly rejected free-meal eligibility because income exceeds 130% FPL. It omitted Minnesota's universal free school meals program, which assigns FREE status regardless of income and eliminates positive reduced-price support." -us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,The model stopped after comparing income with the federal 185% FPL ceiling. It failed to apply Minnesota's universal free meals rule and the resulting FREE-tier precedence over reduced-price support. -us,scenario_032,reduced_price_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model classified the household from the federal 130%–185% poverty band alone. Minnesota universal free meals place the household in the FREE tier, so the reduced-price output is zero." -us,scenario_032,reduced_price_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model inferred positive reduced-price support from the federal income limits and school-age child. It omitted Minnesota's universal free meals program, under which the child receives FREE support instead and no reduced-price support." -us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"The model treated income below the federal reduced-price threshold as guaranteeing a positive reduced-price benefit. Minnesota's universal free meals rule assigns the better FREE tier, which makes reduced-price support zero." -us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,The model used the modeled federal reduced-price income range without applying Minnesota's universal free meals provision. The state provision assigns the FREE tier and supersedes reduced-price eligibility. -us,scenario_032,reduced_price_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"The model correctly identified the 1.56 poverty ratio but incorrectly treated being below 185% FPL as dispositive. Minnesota universal free meals assign the FREE tier regardless of that ratio, leaving reduced-price support at zero." -us,scenario_032,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,The model compared gross income only with the 185% federal threshold. It omitted Minnesota's universal free meals rule and the rule that FREE-tier qualification precludes positive reduced-price support. -us,scenario_032,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer for the requested output. The required value is 0 because Minnesota's universal program assigns the household the superior FREE tier rather than reduced-price support. -us,scenario_032,reduced_price_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model used the 130%–185% federal band to assign reduced-price eligibility. It omitted Minnesota's universal free meals program, which assigns FREE status and supersedes the reduced-price tier." +us,scenario_032,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a household-level “WIC-eligible context” pathway and transferred eligibility to the head based on living with a child. The head is age 50, is not pregnant, postpartum, or breastfeeding, and therefore fails WIC's person-specific categorical requirement; moreover, the only child is age 6 and is not categorically eligible either." +us,scenario_032,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly asserted that Minnesota has no mandatory employee state payroll tax in 2026. It omitted the $127.60 Minnesota paid leave contribution on $29,000 of taxable wages." +us,scenario_032,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly excluded Minnesota's $127.60 paid leave contribution and then submitted $2,175.90 despite repeatedly deriving $2,218.50 in its explanation. Adding the state contribution to the correctly computed federal FICA yields $2,346.10." +us,scenario_032,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,The model incorrectly stated that Minnesota has no mandatory employee state payroll tax. It omitted the $127.60 Minnesota paid leave contribution assessed on the head's wages. +us,scenario_032,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,The model correctly computed federal FICA but incorrectly ruled out a Minnesota employee payroll charge. The missing component is the $127.60 Minnesota paid leave contribution. +us,scenario_032,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"The model stopped after calculating $2,218.50 of federal FICA and incorrectly asserted that no mandatory Minnesota employee payroll tax applied. It omitted the $127.60 Minnesota paid leave contribution." +us,scenario_032,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,The model incorrectly treated federal FICA as the household's complete payroll tax and stated that Minnesota imposed no separate employee charge. Minnesota's paid leave contribution adds $127.60. +us,scenario_032,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly stated that Minnesota's paid family and medical leave produced no employee contribution in 2026. The required Minnesota paid leave contribution is $127.60 on the $29,000 wage base." +us,scenario_032,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model included Minnesota paid leave but applied a 0.35% employee share, producing $101.50. The applicable contribution on $29,000 is $127.60, so total payroll tax is $2,346.10." +us,scenario_032,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,The model calculated only the 7.65% federal FICA charge. It omitted the mandatory $127.60 Minnesota paid leave contribution included in employee payroll tax. +us,scenario_032,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model rounded federal FICA to $2,219 and omitted Minnesota's mandatory employee paid leave contribution. The state component is $127.60, bringing the unrounded total to $2,346.10." +us,scenario_032,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model treated 7.65% federal FICA as the entire employee payroll tax. It omitted the $127.60 Minnesota paid leave contribution on the same $29,000 wage base." +us,scenario_032,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"The model included only Social Security and Medicare taxes. Minnesota's mandatory paid leave contribution adds $127.60 to its $2,218.50 FICA calculation." +us,scenario_032,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,The model equated employee payroll tax with the 7.65% federal FICA rate. It omitted the $127.60 Minnesota paid leave contribution. +us,scenario_032,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required structured-output contract." +us,scenario_032,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model's submitted $4,428.60 does not follow from its own stated 6.2% Social Security and 1.45% Medicare calculation, which produces $2,218.50. It also incorrectly excluded the $127.60 Minnesota paid leave contribution required to reach $2,346.10." +us,scenario_032,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented a 5.24% effective rate and a Social Security wage-base interaction even though $29,000 is fully subject to the 6.2% employee Social Security tax. It also omitted the $127.60 Minnesota paid leave contribution, yielding $1,520 instead of $2,346.10." +us,scenario_032,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"The model correctly calculated $2,218.50 of federal FICA but incorrectly stated that no state payroll tax applied. Minnesota's paid leave contribution adds $127.60." +us,scenario_032,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,The model calculated and rounded federal FICA while omitting Minnesota's mandatory paid leave contribution. The missing state payroll tax is $127.60. +us,scenario_032,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model included only employee Social Security and Medicare taxes. It omitted the $127.60 Minnesota paid leave contribution assessed on the head's wages. +us,scenario_032,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that no mandatory Minnesota employee payroll tax applied. The Minnesota paid leave contribution is $127.60, in addition to $2,218.50 of federal FICA." +us,scenario_032,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,The model stopped after summing Social Security and Medicare taxes. It failed to include Minnesota's $127.60 mandatory employee paid leave contribution. +us,scenario_032,payroll_tax,inkling,llm_error,state_local_rule,False,The model incorrectly stated that no state employee payroll taxes applied. Minnesota's paid leave program contributes an additional $127.60 of employee payroll tax. +us,scenario_032,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required structured-output contract." +us,scenario_032,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model rounded $2,218.50 of federal FICA to $2,219 but omitted the $127.60 Minnesota paid leave contribution. The complete employee payroll-tax total is $2,346.10." +us,scenario_032,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly concluded that Minnesota had no mandatory employee payroll tax because it lacked a disability-insurance tax like other states. Minnesota's paid leave contribution adds $127.60 to the $2,218.50 federal FICA amount." +us,scenario_032,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"The model omitted Minnesota's $127.60 paid leave contribution and also misstated Medicare tax as $419.50 rather than $420.50. Its submitted $2,218.50 reflects federal FICA alone; the complete total is $2,346.10." +us,scenario_032,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the federal 185% FPL reduced-price ceiling without applying Minnesota's universal free school meals program. That state pathway assigns the household to the superior FREE tier, so reduced-price support is zero." +us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model classified the household using only the federal 130%-to-185% FPL bands and concluded that income above 130% barred free meals. Minnesota provides universal free meals regardless of income, which assigns the FREE tier and supersedes reduced-price eligibility." +us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model used the federal income band as the complete eligibility test and placed the child in the reduced-price tier. It omitted Minnesota's universal free-meals pathway, under which the household receives the FREE tier and therefore no reduced-price support." +us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated an income ratio between 130% and 185% FPL as dispositive. Minnesota's universal free school meals program instead places the household in the FREE tier, which eliminates reduced-price support." +us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model repeatedly applied the ordinary federal income bands and asserted that income above 130% FPL prevented free-meal eligibility. It omitted Minnesota's universal free school meals program, which grants the FREE tier regardless of income and supersedes reduced-price meals." +us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model stopped after comparing income with the 185% FPL ceiling. It failed to apply Minnesota's universal free school meals program, which assigns the superior FREE tier and leaves no positive reduced-price benefit." +us,scenario_032,reduced_price_school_meals_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model relied exclusively on the federal 130%-to-185% FPL reduced-price range. Minnesota's universal free-meals rule places this household in the FREE tier regardless of that ratio, so reduced-price eligibility is false." +us,scenario_032,reduced_price_school_meals_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the federal reduced-price and free-meal income limits as the only tiering rules. It omitted Minnesota's universal free school meals program, which produces positive free-meal support and supersedes reduced-price support." +us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model inferred positive reduced-price support solely from income below the federal ceiling. Minnesota's universal free-meals pathway assigns the FREE tier first, so the reduced-price output remains zero." +us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model equated membership in the ordinary federal reduced-price income range with a positive reduced-price benefit. It omitted the Minnesota universal-free classification, which supersedes that tier and yields no reduced-price support." +us,scenario_032,reduced_price_school_meals_eligible,grok-4.5,llm_error,categorical_eligibility,False,"The model correctly identified the 1.56 poverty ratio but incorrectly treated the 185% FPL test as controlling. Minnesota's universal free school meals program assigns the FREE tier regardless of income, preventing positive reduced-price support." +us,scenario_032,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model stopped at the finding that the household's income ratio was below 185% FPL. It failed to apply Minnesota's universal free school meals program, under which the household receives the superior FREE tier rather than reduced-price meals." +us,scenario_032,reduced_price_school_meals_eligible,inkling,llm_error,categorical_eligibility,False,"The model classified the household from the federal 130%-to-185% FPL band alone. Minnesota's universal free-meals rule places the household in the FREE tier, which supersedes and zeros out reduced-price support." +us,scenario_032,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for the requested output, so it failed the required structured-output contract." +us,scenario_032,reduced_price_school_meals_eligible,kimi-k3,llm_error,categorical_eligibility,False,"The model used the ordinary federal income bands and school-age status but omitted Minnesota's universal free school meals program. The state rule assigns the FREE tier at the household's 1.56 poverty ratio, so reduced-price support is not positive." us,scenario_032,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_032,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly treated Minnesota broad-based categorical eligibility as waiving the controlling gross-income test. Its own net-income calculation produced a negative allotment, but it then discarded that result and invented $493 per month using unlisted shelter and utility considerations." -us,scenario_032,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly used Minnesota broad-based categorical eligibility to convert income above the applicable SNAP limit into eligibility. It also calculated that 30% of net income exceeded the maximum allotment, then replaced the resulting $0 with an unsupported $476 monthly benefit; a minimum benefit does not turn this ineligible household into an eligible one." -us,scenario_032,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Minnesota’s broad-based categorical eligibility made the household eligible at 165% of FPL. The household fails the applicable gross-income eligibility test, so the earned-income and standard deductions do not yield a $127 monthly allotment." -us,scenario_032,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model recognized that gross income exceeded its cited limit but nevertheless assumed categorical eligibility and awarded a benefit after deductions. It also treated the presence of a young child as supporting a positive allotment, although a child does not override the SNAP income test." -us,scenario_032,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model asserted that net income qualified without applying the controlling gross-income limit. Annual gross income of $42,664 exceeds that limit, yielding $0 before any allotment calculation." -us,scenario_032,snap,grok-4.3,llm_error,thresholds_rates,False,"The answer implies that the model applied an allotment formula without first enforcing SNAP’s income eligibility threshold. The household’s $42,664 gross income exceeds the applicable limit, so no $200 monthly benefit is payable." -us,scenario_032,snap,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model invented a fourth household member from the head’s Social Security dependent-benefit receipt even though the prompt lists exactly three people. That inflated the SNAP income limits and maximum allotment; the actual three-person household’s $42,664 gross income exceeds the applicable eligibility limit." -us,scenario_032,spouse_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model checked only the spouse's age and household income against a CHIP threshold. It omitted the prior Medicaid determination: the spouse is Medicaid-eligible under the OLDER_CHILD category, which makes the spouse ineligible for CHIP." -us,scenario_032,spouse_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,The model treated being under 19 and below its stated Minnesota CHIP income threshold as sufficient for CHIP eligibility. It failed to apply the exclusion for children already eligible for Medicaid; the spouse qualifies for Medicaid under the OLDER_CHILD category. -us,scenario_032,spouse_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model applied age, income, and absence-of-other-coverage checks but never tested Medicaid eligibility. The spouse's Medicaid eligibility under the OLDER_CHILD category precludes CHIP regardless of being under 19 and below the CHIP income threshold." -us,scenario_032,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no spouse_chip_eligible output or explanation, violating the required structured-output contract." -us,scenario_032,spouse_medicaid_eligible,claude-opus-5,llm_error,categorical_eligibility,False,The model applied the 138% FPL adult Medicaid threshold to an 18-year-old whom Minnesota classifies in the OLDER_CHILD category. The spouse’s MAGI of 1.56 times FPL satisfies the applicable older-child threshold. -us,scenario_032,spouse_medicaid_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model reused the adult Medicaid income limit instead of evaluating the spouse under Minnesota’s OLDER_CHILD pathway. At age 18 and 1.56 times FPL, the spouse meets the older-child income test." -us,scenario_032,spouse_medicaid_eligible,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model treated the spouse as an expansion adult subject to the 138% FPL ceiling. Minnesota instead places this 18-year-old in the OLDER_CHILD category, whose income threshold includes MAGI at 1.56 times FPL." -us,scenario_032,spouse_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model compared an incorrectly constructed $42,664 household-income total with the adult expansion threshold and never applied the OLDER_CHILD pathway. The governing computation uses the spouse’s MAGI income level of 1.56 times FPL, which passes Minnesota’s older-child threshold." -us,scenario_032,spouse_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model assumed the 18-year-old needed to establish an adult eligibility category. Age 18 itself places the spouse in Minnesota’s OLDER_CHILD category, and MAGI at 1.56 times FPL meets that category’s income limit." -us,scenario_032,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required disability, pregnancy, SSI, or TANF eligibility and omitted the age-based OLDER_CHILD pathway. The spouse qualifies through that pathway at age 18 with MAGI of 1.56 times FPL." -us,scenario_032,spouse_medicaid_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model applied an unspecified adult income limit and counted wages plus Social Security dependent benefits as the controlling household-income measure. Minnesota evaluates this 18-year-old under the OLDER_CHILD MAGI pathway, where the traced 1.56-times-FPL income level is within the threshold." -us,scenario_032,spouse_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model compared the spouse with Minnesota’s adult Medicaid income limit instead of the OLDER_CHILD limit. The spouse is age 18 and the applicable older-child pathway accepts MAGI of 1.56 times FPL. -us,scenario_032,spouse_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model omitted the required spouse_medicaid_eligible output entirely. The required computation classifies the age-18 spouse as OLDER_CHILD and yields eligibility at 1.56 times FPL. +us,scenario_032,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly used Minnesota broad-based categorical eligibility to bypass the failed SNAP gross-income screen, then contradicted its own calculation showing that net income produced a negative allotment. It replaced that result with an unsupported $493 monthly benefit despite no listed shelter expense or other deduction capable of generating it." +us,scenario_032,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated Minnesota broad-based categorical eligibility as sufficient to pass the household through the gross-income screen. Its own max-allotment-minus-30%-of-net-income calculation was negative, but it then invented a $476 monthly benefit; SNAP's minimum benefit does not convert a negative allotment for this ineligible three-person household into that amount." +us,scenario_032,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Minnesota's 165% broad-based categorical-eligibility threshold made the household eligible despite its $42,664 gross income. The gross-income screen yields no eligibility, so the model should never have proceeded to calculate a $127 monthly allotment." +us,scenario_032,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model explicitly recognized that gross income exceeded its stated eligibility threshold but nevertheless awarded benefits on the basis that categorical eligibility and deductions could preserve a small allotment. Deductions used in the net-income calculation do not reverse the failed gross-income eligibility screen, and the young child's presence does not independently create SNAP eligibility." +us,scenario_032,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The answer's bare assertion that net income qualifies skips the controlling gross-income screen. The household's $42,664 annual gross income exceeds the applicable limit, ending SNAP eligibility before any positive allotment calculation." +us,scenario_032,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model calculated an allotment from household size and income without first enforcing the gross-income eligibility limit. Applying that screen to the three-person household's $42,664 annual gross income yields no SNAP eligibility and therefore no benefit." +us,scenario_032,snap,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model invented a fourth household member from the head's receipt of Social Security dependent benefits, even though the prompt lists exactly two adults and one child. That inflated household size and its associated income limits and allotment; for the actual three-person unit, $42,664 of gross income fails the SNAP gross-income screen." +us,scenario_032,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The unsupported statement that the household qualifies omits the decisive gross-income eligibility test. Wages and Social Security dependent benefits total $42,664 annually, above the applicable limit, so no allotment is calculated." +us,scenario_032,spouse_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model checked only the spouse's age and household income against a CHIP threshold. It omitted the prior Medicaid eligibility determination: the spouse is Medicaid-eligible under the OLDER_CHILD category, which automatically makes the spouse ineligible for CHIP." +us,scenario_032,spouse_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated being under 19 and below Minnesota's CHIP income threshold as sufficient for CHIP eligibility. It failed to apply the Medicaid-exclusion step: the spouse qualifies for Medicaid as an OLDER_CHILD and therefore cannot qualify for CHIP. +us,scenario_032,spouse_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model concluded that age, income, and lack of other listed health coverage established CHIP eligibility. It never tested the controlling exclusion for Medicaid-eligible children; the spouse is Medicaid-eligible under the OLDER_CHILD category, so CHIP eligibility is false." +us,scenario_032,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no spouse_chip_eligible value or explanation, violating the required structured-output contract. The required result is 0 because the spouse's Medicaid eligibility under the OLDER_CHILD category precludes CHIP eligibility." +us,scenario_032,spouse_medicaid_eligible,claude-opus-5,llm_error,categorical_eligibility,False,The model applied Minnesota's 138% FPL adult Medicaid threshold to an 18-year-old who qualifies through the OLDER_CHILD category. The applicable older-child threshold admits the spouse at 1.56 times FPL. +us,scenario_032,spouse_medicaid_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,The model reused the adult Medicaid income limit instead of evaluating the spouse under Minnesota's OLDER_CHILD pathway. The spouse is age 18 and eligible under that category at 1.56 times FPL. +us,scenario_032,spouse_medicaid_eligible,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model treated the 138% FPL Medicaid expansion limit as controlling for the spouse. Minnesota classifies the 18-year-old spouse as an OLDER_CHILD, whose applicable income threshold includes 1.56 times FPL." +us,scenario_032,spouse_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model compared its $42,664 household-income estimate with the 138% FPL adult threshold and stopped there. It failed to apply Minnesota's OLDER_CHILD category to the age-18 spouse, for whom the engine-derived MAGI level is 1.56 times FPL and within the category threshold." +us,scenario_032,spouse_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model searched only for a qualifying adult category and overlooked that an 18-year-old qualifies under Minnesota's OLDER_CHILD category. That pathway covers the spouse at a MAGI level of 1.56 times FPL. +us,scenario_032,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required disability, pregnancy, SSI, or TANF rather than evaluating age-based child Medicaid eligibility. At age 18, the spouse falls within Minnesota's OLDER_CHILD category and meets its income threshold at 1.56 times FPL." +us,scenario_032,spouse_medicaid_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model applied an unspecified Minnesota adult income limit after summing wages and Social Security benefits. It omitted the OLDER_CHILD eligibility pathway for the age-18 spouse, which finds the relevant MAGI level of 1.56 times FPL within the category threshold." +us,scenario_032,spouse_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model used Minnesota's adult Medicaid income limit for the age-18 spouse. The controlling pathway is OLDER_CHILD, under which 1.56 times FPL satisfies the income threshold." +us,scenario_032,spouse_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer for spouse_medicaid_eligible, violating the required output contract. The required eligibility value follows from the spouse's OLDER_CHILD category and qualifying MAGI level." +us,scenario_032,spouse_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The model's blanket denial failed to apply Minnesota's OLDER_CHILD category to the 18-year-old spouse. The spouse's MAGI level of 1.56 times FPL falls within that category's income threshold. us,scenario_032,spouse_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_032,spouse_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly reasoned that the spouse lacks pregnancy, postpartum, or breastfeeding status and that the age-6 child is not WIC-categorical, but it submitted the opposite numeric value. Its error is an answer-contract inconsistency: the explanation supports ineligibility while the output says eligible." -us,scenario_032,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated being an 18-year-old woman of childbearing age in a low-income household with a child as WIC categorical eligibility. WIC does not cover adult women on that basis; the spouse needed listed pregnancy, postpartum, or breastfeeding status, and the age-6 child does not create WIC eligibility for her." -us,scenario_032,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model misapplied WIC categorical eligibility by presuming that an 18-year-old woman qualifies whenever household income is under the WIC limit. The spouse has no listed pregnancy, postpartum, or breastfeeding status, and WIC child eligibility only applies to children under age 5, so income eligibility cannot make her eligible." -us,scenario_032,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an understated Minnesota joint standard deduction of $15,107 and a separate exemption amount of $14,916, leaving $893 of taxable income. PolicyEngine's Minnesota calculation eliminates taxable income for this household, so the model's $48 is the direct result of applying the wrong Minnesota deduction/exemption structure." -us,scenario_032,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave only a generic Minnesota bracket calculation and produced a positive tax, which means it left taxable income after deductions and exemptions. The correct Minnesota computation reduces this low-income joint filer with one dependent to zero state income tax before refundable credits." -us,scenario_032,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an approximate Minnesota MFJ standard deduction of $30,699 against AGI of $30,916, leaving $217 taxable income taxed at 5.35%. That shortcut omitted the full Minnesota deduction/exemption calculation that removes the remaining taxable income and yields zero tax." -us,scenario_032,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable value or explanation for this output. This is a missing-output contract failure rather than a substantive Minnesota tax calculation. -us,scenario_032,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the $13,664 Social Security dependent benefits as fully taxable for Minnesota in its simplified calculation and then applied approximate deductions and credits. PolicyEngine's Minnesota computation for this household reaches zero state income tax before refundable credits, so the model's positive result comes from overstating taxable income and using non-engine approximations." -us,scenario_032,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model estimated separate child and working-family components using approximate amounts and treated the dependent Social Security benefits as merely preserving most of the credit. The 2026 Minnesota CWFC formula applied to AGI of $30,916 and adjusted earnings of $29,000 yields $2,187.60, not $2,337." -us,scenario_032,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Minnesota has no refundable individual income-tax credits. Minnesota's refundable CWFC applies to this working filer with one eligible child and produces $2,187.60." -us,scenario_032,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a fixed $1,750 child credit and an approximate $410 Working Family Credit instead of the 2026 CWFC parameters. Applying the program formula to AGI of $30,916 and adjusted earnings of $29,000 produces $2,187.60." -us,scenario_032,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model submitted only the assumed $1,750 Minnesota child credit after acknowledging that the Working Family Credit adds another amount. The combined 2026 CWFC computation for this filer and child equals $2,187.60." -us,scenario_032,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model approximated the shared CWFC phaseout and rounded the result to $2,185. The exact 2026 calculation using $30,916 of AGI and $29,000 of adjusted earnings yields $2,187.60." -us,scenario_032,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model treated Minnesota's Working Family Credit as approximately 35% of federal EITC, incorrectly added 85% of the dependent Social Security benefits to AGI, and omitted the child-credit portion of CWFC. The applicable inputs are AGI of $30,916, adjusted earnings of $29,000, and one eligible child, which produce $2,187.60." -us,scenario_032,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model wrongly treated rent, property tax, education expenses, or other verified expenses as prerequisites for every Minnesota refundable credit. Wages of $29,000 and one eligible child independently establish the CWFC pathway, producing $2,187.60." -us,scenario_032,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an inflation-estimated $1,912 child credit and calculated the Working Family Credit as 33% of federal EITC. Minnesota's 2026 CWFC is not computed by adding those two approximations; its formula yields $2,187.60 from the stated AGI, adjusted earnings, and child count." -us,scenario_032,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model imposed an unsupported phaseout that reduced the child component to $983.60 and guessed a $550 Working Family Credit. The actual 2026 CWFC calculation at $30,916 AGI and $29,000 adjusted earnings equals $2,187.60." -us,scenario_032,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model failed to recognize the Minnesota CWFC eligibility pathway for a working filer with one eligible child. Those facts generate a refundable credit of $2,187.60." -us,scenario_032,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model invented a flat $700 Working Family Credit and assumed no phaseout, then added it to a $1,750 child credit. The 2026 CWFC formula uses both $30,916 of AGI and $29,000 of adjusted earnings and yields $2,187.60." -us,scenario_032,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assigned an unsupported approximate $1,200 Working Family Credit on top of a $1,750 child credit. The combined 2026 CWFC computation instead equals $2,187.60." -us,scenario_032,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model counted only a $1,750 Minnesota child credit and omitted the working-families portion of the combined CWFC calculation. The full refundable CWFC amount is $2,187.60." -us,scenario_032,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model inflation-adjusted the old $1,750 child-credit amount to $1,912 and declared that no other refundable credit applied. It omitted the working-families component and failed to apply the unified 2026 CWFC formula, which yields $2,187.60." -us,scenario_032,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to identify Minnesota's refundable CWFC despite $29,000 of adjusted earnings and one eligible child. The applicable credit equals $2,187.60." -us,scenario_032,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required the prompt to supply a precomputed credit amount or additional filer details. The listed wages, household filing facts, and eligible child supply the inputs needed for Minnesota's CWFC, which equals $2,187.60." -us,scenario_032,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a guessed $350 Working Family Credit and a fixed $1,750 child credit. Minnesota's 2026 combined CWFC formula produces $2,187.60 from $30,916 of AGI and $29,000 of adjusted earnings." -us,scenario_032,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model incorrectly computed Minnesota's Working Family Credit as 25% of an estimated federal EITC and added a fixed $1,750 child credit. The state CWFC uses its own 2026 parameters and yields $2,187.60." -us,scenario_032,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model assigned an unsupported $1,675 Working Family Credit in addition to a $1,750 child credit. Applying Minnesota's 2026 CWFC formula to the household's actual AGI and adjusted earnings yields $2,187.60." -us,scenario_032,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The submitted $3,572 implies substantially overstated CWFC components and does not follow the applicable income calculation. With one eligible child, $30,916 of AGI, and $29,000 of adjusted earnings, the credit is $2,187.60." -us,scenario_032,state_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model inserted a generic $500 estimate for the Working Family Credit without calculating the full Child and Working Families Credits amount. The 2026 formula produces $2,187.60." -us,scenario_032,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly set Minnesota's Working Family Credit to 40% of federal EITC and added a fixed $1,750 child credit. The state's 2026 CWFC formula is not that federal-EITC percentage calculation and yields $2,187.60." -us,scenario_032,state_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly calculated the Working Family Credit as 33% of federal EITC and added a fixed $1,750 child credit. Minnesota's own 2026 CWFC parameters produce $2,187.60." -us,scenario_032,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric value or explanation for the requested output. -us,scenario_032,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model assumed a full flat $1,250 Working Family Credit and a full $1,750 child credit with no income adjustment. The 2026 CWFC computation using AGI of $30,916 and adjusted earnings of $29,000 yields $2,187.60." -us,scenario_032,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model wrongly treated the spouse's lack of earnings and household income as disqualifying. The filer has $29,000 of adjusted earnings, meets the working-families requirements, and has one eligible child, generating $2,187.60." -us,scenario_032,state_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model approximated the Working Family Credit from federal EITC percentages and omitted the child-credit portion of Minnesota's combined CWFC. The correct state formula produces $2,187.60." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly derived taxable income of about $35,708 and correctly computed tax of about $3,789, then replaced that result with $3,699 under an unsupported “PolicyEngine-style rounding” adjustment. Ordinary rounding cannot reduce the bracket calculation by $90." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated the $11,400 business loss as a capital loss limited to $3,000, subtracted the standard deduction while labeling the result AGI, and omitted the two senior deductions. The business loss is fully included above the line, and total standard, age-based, and senior deductions equal $47,500." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model reached the correct taxable income of about $35,708 after applying the senior deductions, but then abandoned its own bracket result of about $3,802 and submitted $3,416 based on unspecified “different bracket estimates.” The applicable 2026 brackets produce $3,788.99." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $243 state and local tax refund in income and used understated 2026 standard and age-based deductions. It then computed tax near $3,798 from its own figures but submitted $4,575 without any corresponding computation." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model incorrectly included the $243 state-tax refund, producing taxable income of $34,951 instead of $35,708.23. It also mis-added its own bracket calculation: $2,480 plus $1,218 equals about $3,698, not $3,080." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model included the $243 refund and omitted the $12,000 senior deduction available to the two taxpayers age 65 or older. Those errors left taxable income at $50,251 instead of $35,708.23 and drove the excessive tax estimate." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model initially omitted the $12,000 senior deduction and calculated tax of about $5,241 on taxable income near $47,808. It then submitted $3,800 by invoking unspecified adjustments and credits, rather than applying the senior deduction and exact 2026 brackets to $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used 2024 rules for a 2026 return, included the $243 refund, and omitted the $12,000 senior deduction. This overstated taxable income as $50,351 instead of $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA provisions expired for 2026 and therefore substituted a reduced standard deduction, personal exemptions, and a 15% second bracket. The applicable 2026 rules retain the larger deduction structure, include $12,000 of senior deductions, and tax the resulting $35,708.23 at 10% and 12%." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The stated AGI of $78,818 and taxable income of $46,318 do not incorporate the traced income and deduction calculation. Taxable Social Security, the full self-employment loss, the HSA deduction, and $47,500 of standard and senior deductions produce AGI of $83,208.23 and taxable income of $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The submitted $6,762 does not result from the traced taxable income of $35,708.23 under the 2026 joint brackets. Its unexplained estimate reflects a substantially overstated tax base, omitting the full standard, age-based, and senior deduction total of $47,500." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly assumed a 2026 TCJA reversion and used a smaller standard deduction, restored personal exemptions, and reverted tax brackets. The actual deduction total is $47,500, leaving $35,708.23 taxed under the 10% and 12% brackets." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model applied only $34,000 of standard and age-based deductions and omitted the additional $12,000 senior deduction. This overstated taxable income by roughly $13,500 and produced excessive tax." -us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included the $243 refund in AGI and omitted the $12,000 senior deduction. Its $34,030 deduction estimate therefore left taxable income at $49,421 rather than the traced $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The submitted $3,677 does not follow from the stated placement of taxable income in the 12% bracket. With taxable income of $35,708.23 after the $47,500 deduction total, the exact 2026 bracket calculation is $3,788.99; mortgage interest and the SALT refund do not supply the unexplained reduction." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated Social Security as nontaxable and concluded that the deductions eliminated all liability. At this provisional income, $16,320 of Social Security is taxable, and AGI of $83,208.23 minus $47,500 leaves $35,708.23 subject to tax." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model assumed post-TCJA reversion, included the $243 refund, used only $29,900 of deductions and exemptions, and applied a 15% second bracket. The 2026 computation instead uses $47,500 of deductions and the 10% and 12% brackets." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used taxable income of $35,951, exactly $242.77 above the traced $35,708.23, reflecting inclusion of the $243 state-tax refund. That refund is excluded from IRS gross income here, so the correct bracket tax is $3,788.99." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model explicitly included the $243 state-tax refund, raising taxable income from $35,708.23 to approximately $35,951. Removing that refund reduces the 12%-bracket tax by about $29.13 to $3,788.99." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $3,818.12 result is the bracket tax on approximately $35,951, which is $242.77 above the traced taxable income. The excess corresponds to including the $243 state-tax refund that is excluded from gross income in this calculation." -us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $5,523 answer corresponds to taxing income near $50,000 rather than the traced $35,708.23. The model's shortcut omitted the two $6,000 senior deductions from the combined $47,500 deduction total." -us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model included the $243 refund in AGI and applied only $34,030 of standard and age-based deductions, omitting the $12,000 senior deduction. Those choices overstated taxable income as $49,421 instead of $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model included the $243 refund and omitted the two $6,000 senior deductions, using only $34,196 of deductions. This left taxable income near $49,255 rather than $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required structured-output contract before any substantive tax calculation could be assessed. -us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model included the $243 refund, then improperly deducted $1,225 of auto-loan interest and invented a $333.33 qualified-overtime deduction from annual wages and scheduled hours. Neither deduction applies from the supplied facts, and excluding the refund yields taxable income of $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model omitted the $11,400 self-employment loss, mishandled Social Security by first including the full benefit and then adding its taxable portion, and omitted the $12,000 senior deduction. It then submitted a negative income-tax liability despite computing roughly $10,800 of tax and despite this output being tax before refundable credits." -us,scenario_033,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $11,400 self-employment loss when computing AGI and omitted the $12,000 senior deduction. Those two omissions overstated taxable income by $23,400 before smaller parameter differences, producing the excessive tax estimate." +us,scenario_032,spouse_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly stated that the spouse lacks pregnancy, postpartum, and breastfeeding status and therefore is not categorically eligible, but then submitted 1 in direct contradiction to its reasoning. WIC adjunctive eligibility does not create categorical status through another household member." +us,scenario_032,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated being an 18-year-old woman of childbearing age in a low-income household with a child as a WIC category. WIC requires the adult woman herself to be pregnant, breastfeeding, or postpartum, and none of those statuses is present." +us,scenario_032,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model presumed that a woman of childbearing age becomes categorically eligible once household income passes the WIC income test. Income eligibility cannot replace the separate requirement that she be pregnant, breastfeeding, or postpartum, and the prompt makes all three unlisted statuses false." +us,scenario_032,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $15,107 standard deduction, understating the deduction available to a married joint filer, and therefore manufactured $893 of taxable income. The correct joint-filer deductions and exemptions eliminate Minnesota taxable income, so no 5.35% tax arises." +us,scenario_032,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The $242 answer implies that the model retained about $4,523 of taxable income at Minnesota's 5.35% first-bracket rate. It failed to apply the full joint-filer deductions and exemptions that reduce Minnesota taxable income to zero." +us,scenario_032,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model stopped after subtracting an approximate joint standard deduction and taxed the remaining $217. It omitted the applicable exemption subtraction, which eliminates that residual taxable income before the 5.35% rate is applied." +us,scenario_032,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied neither the required numeric value nor an explanation for state_income_tax_before_refundable_credits. This is a missing-output contract failure. +us,scenario_032,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated all $13,664 of Social Security dependent benefits as Minnesota taxable income, overstating the state income base. It also failed to carry the applicable joint-filer deductions and exemptions through to zero taxable income before applying Minnesota's 5.35% rate." +us,scenario_032,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model estimated separate child and working-family components and treated the household as receiving nearly all of both. Applying the 2026 Minnesota CWFC computation to $30,916 AGI and $29,000 adjusted earnings yields $2,187.60, not $2,337." +us,scenario_032,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Minnesota has no refundable individual income tax credits. The household qualifies for Minnesota's refundable CWFC through its one eligible child and $29,000 of adjusted earnings." +us,scenario_032,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $410 Working Family Credit and added it to a presumed $1,750 child credit. Minnesota's 2026 CWFC formula using $30,916 AGI and $29,000 adjusted earnings produces $2,187.60." +us,scenario_032,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model counted only a presumed $1,750 Minnesota child credit and omitted the working-families portion of the CWFC. The combined 2026 calculation produces $2,187.60." +us,scenario_032,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model approximated the shared CWFC phaseout and understated the result by $2.60. The exact 2026 computation with $30,916 AGI, $29,000 adjusted earnings, and one eligible child is $2,187.60." +us,scenario_032,state_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly included 85% of the dependent Social Security benefits in AGI and then approximated Minnesota's credit as 35% of federal EITC. The applicable inputs are $30,916 AGI and $29,000 filer adjusted earnings under Minnesota's own CWFC formula, yielding $2,187.60." +us,scenario_032,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model wrongly required rent, property tax, education expenses, or additional verification despite the listed wages and qualifying child. Those facts satisfy the CWFC pathway and generate $2,187.60." +us,scenario_032,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model substituted a full inflation-adjusted child credit plus 33% of a federal EITC estimate. Minnesota's 2026 CWFC uses its own integrated computation and yields $2,187.60 from the stated AGI, earnings, and child." +us,scenario_032,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model applied an unsupported child-credit phaseout despite AGI of $30,916 and assigned an estimated $550 working-family amount. The exact 2026 CWFC calculation is $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted Minnesota's CWFC entirely. One eligible child and $29,000 of adjusted earnings qualify the household for $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model invented a flat $700 working-family base and added it to a $1,750 child amount. The 2026 Minnesota CWFC computation produces $2,187.60 rather than $2,450." +us,scenario_032,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assigned an unsupported approximate $1,200 Working Family Credit. Minnesota's 2026 CWFC schedule applied to the household's AGI and adjusted earnings yields $2,187.60 in total." +us,scenario_032,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model counted only a $1,750 child-credit amount and omitted the working-families portion of Minnesota's CWFC. The combined refundable credit is $2,187.60." +us,scenario_032,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model independently inflated the historical child-credit amount and incorrectly concluded that no other refundable credit applied. Minnesota's 2026 CWFC calculation includes the working-families component and totals $2,187.60." +us,scenario_032,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to identify Minnesota's refundable CWFC. The listed child and earnings establish eligibility and produce $2,187.60." +us,scenario_032,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly supplied credit amount as absence of eligibility. The household facts supply the qualifying child, AGI, and adjusted earnings needed to calculate a $2,187.60 CWFC." +us,scenario_032,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a flat $350 working-family amount rather than the 2026 Minnesota CWFC calculation. The correct computation from $30,916 AGI and $29,000 adjusted earnings totals $2,187.60." +us,scenario_032,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model incorrectly calculated Minnesota's working-family amount as 25% of an estimated federal EITC. The state CWFC has its own 2026 formula, which yields $2,187.60 in total." +us,scenario_032,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model assigned an unsupported $1,675 Working Family Credit and added it to a presumed $1,750 child amount. The Minnesota CWFC computation produces $2,187.60." +us,scenario_032,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model's $3,572 answer implies materially overstated CWFC parameters or failure to apply the applicable income computation. Using one child, $30,916 AGI, and $29,000 adjusted earnings yields $2,187.60." +us,scenario_032,state_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an unsupported $500 estimate for Minnesota refundable credits without calculating the child component of the CWFC. The complete 2026 computation yields $2,187.60." +us,scenario_032,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly set the Working Family Credit equal to 40% of estimated federal EITC and added a separate full child credit. Minnesota's own 2026 CWFC calculation yields $2,187.60." +us,scenario_032,state_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly used 33% of estimated federal EITC as the Minnesota Working Family Credit. Applying the 2026 state CWFC formula to the household's actual inputs yields $2,187.60." +us,scenario_032,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model inserted an estimated $1,300 Working Family Credit rather than applying Minnesota's 2026 CWFC formula. The exact combined refundable amount is $2,187.60." +us,scenario_032,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, so it failed the required output contract." +us,scenario_032,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model assumed a full $1,250 Working Family Credit and a separate full $1,750 child credit. The 2026 Minnesota CWFC computation based on $30,916 AGI and $29,000 adjusted earnings totals $2,187.60." +us,scenario_032,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly required the spouse to have earnings and incorrectly placed the household above the eligibility thresholds. The filer's $29,000 of adjusted earnings and one eligible child qualify the tax unit for a $2,187.60 CWFC." +us,scenario_032,state_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model approximated Minnesota's credit from federal EITC percentages and omitted the child component of the CWFC computation. The complete 2026 state calculation yields $2,187.60." +us,scenario_032,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a $241 child and dependent care credit despite no listed care expenses and failed to calculate Minnesota's qualifying-child CWFC correctly. The refundable amount derives entirely from the CWFC and equals $2,187.60." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own derivation reaches $35,708 of taxable income and $3,789 of tax, then replaces that result with $3,699 based on nonexistent “PolicyEngine-style rounding.” Rounding cannot reduce the computed tax by $90." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated the $11,400 self-employment loss as a capital loss subject to the $3,000 limit, deducted the standard deduction while constructing AGI, and omitted the age and senior deductions. A business loss reduces income in full, and the combined deductions total $47,500 after AGI is calculated." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly reached approximately $35,708 of taxable income and initially computed tax near $3,802 using estimated brackets, but then submitted $3,416 without a supporting calculation. The actual 2026 bracket cutoff yields $3,788.99." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model included the $243 state tax refund even though it is excluded under the tax-benefit rule, then introduced a separate $1,000 arithmetic error when summing non-Social-Security income. Its stated taxable income supports tax near $3,798, not the submitted $4,575." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $243 state tax refund, producing $34,951 instead of $35,708.23 of taxable income, and then mis-added the bracket tax: $2,480 plus $1,218 equals about $3,698, not $3,080." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model included the nontaxable $243 state tax refund and omitted the temporary $12,000 senior deduction for the two spouses aged 65 or older. Those errors left taxable income at $50,251 instead of $35,708.23." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model's calculation stopped after the ordinary standard and age additions and omitted the $12,000 senior deduction, yielding $47,808 of taxable income. It then asserted $3,800 through unspecified adjustments rather than applying the deductions and brackets that produce $3,788.99." +us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applied 2024 rules to a 2026 return, included the $243 refund, and omitted the $12,000 senior deduction. The 2026 combined deduction is $47,500, leaving $35,708.23 taxable rather than $50,351." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA provisions expired and applied a reduced standard deduction, restored personal exemptions, and a 15% bracket. The applicable 2026 rules instead provide $47,500 of standard, age, and senior deductions and retain the 10% and 12% brackets for this taxable income." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted explanation uses an unsupported AGI of $78,818 and taxable income of $46,318. The income components produce AGI of $83,208.23, and the $47,500 combined deductions reduce taxable income to $35,708.23." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"The model supplied no derivation beyond saying it taxed estimated taxable income. The correct traced taxable income is $35,708.23, whose 2026 married-joint bracket tax is $3,788.99, so $6,762 reflects an unsupported taxable-income or rate estimate." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assumed a post-TCJA reversion, substituted a reduced standard deduction and personal exemptions, and applied reverted brackets. The applicable 2026 calculation uses $47,500 of standard, age, and senior deductions and the current 10% and 12% bracket schedule." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used only $34,000 of deductions and omitted $13,500 of the applicable standard, age, and senior deductions. The full $47,500 deduction reduces taxable income from $83,208.23 to $35,708.23." +us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included the $243 state tax refund and omitted the $12,000 senior deduction, using only about $34,030 of deductions. The refund is excluded and total deductions are $47,500." +us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model did not state the taxable-income figure, deduction total, or bracket computation supporting $3,677. The traced calculation yields $35,708.23 of taxable income and $3,788.99 after applying the 2026 joint brackets." +us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated Social Security as wholly nontaxable and asserted that deductions eliminated taxable income. In fact, $16,320 of Social Security is taxable and the $47,500 deductions leave $35,708.23 subject to tax." +us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a post-TCJA regime with personal exemptions and a 15% bracket, while also including the $243 refund. The applicable rules exclude that refund, allow $47,500 in combined deductions, and tax the resulting income at 10% and 12%." +us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $35,951, exactly reflecting inclusion of the $243 state tax refund. Excluding that refund produces $35,708.23 of taxable income and lowers the bracket tax to $3,788.99." +us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model expressly included the $243 state tax refund and therefore used taxable income of $35,951. The refund is excluded, so taxable income is $35,708.23 and tax is $3,788.99." +us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The submitted $3,818.12 corresponds to taxing approximately $35,951, which includes the $243 state tax refund. Excluding that refund yields the traced $35,708.23 taxable income and $3,788.99 tax." +us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model gave no figures for its taxable-income derivation, but $5,523 is consistent with stopping at the ordinary standard and age additions while omitting the $12,000 senior deduction. The full $47,500 deduction leaves $35,708.23 taxable." +us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model included the $243 state tax refund and omitted the $12,000 senior deduction, producing $49,421 of taxable income. The refund is excluded and the full deduction is $47,500, yielding $35,708.23." +us,scenario_033,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model included the $243 state tax refund and allowed only $34,196 of deductions, omitting the $12,000 senior deduction and misstating the remaining deduction amount. The correct AGI is $83,208.23 and combined deductions are $47,500." +us,scenario_033,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model deducted $1,225 of auto-loan interest without facts establishing a qualifying vehicle and also used unsupported deduction amounts. No auto-loan-interest deduction applies here; AGI of $83,208.23 less $47,500 yields $35,708.23 taxable." +us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for this requested variable. +us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $243 state tax refund and deducted both $1,225 of auto-loan interest and $333.33 of purported overtime premium. The facts establish neither a qualifying auto-interest deduction nor overtime pay above the straight-time rate, so taxable income remains $35,708.23." +us,scenario_033,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model ignored the $11,400 self-employment loss, mishandled Social Security in its income totals, omitted the senior deduction, and then submitted a negative income-tax liability despite computing roughly $10,800 of tax. Federal income tax before refundable credits cannot represent a refund, and the traced liability is $3,788.99." +us,scenario_033,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $11,400 self-employment loss from AGI even after identifying it as a net loss, and it omitted the $12,000 senior deduction. The loss reduces income in full, and total standard, age, and senior deductions equal $47,500." +us,scenario_033,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and a Social Security subtraction reduced taxable income to zero. Taxable Social Security is $16,320, and after the $47,500 combined deductions the household still has $35,708.23 of taxable income." us,scenario_033,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_033,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly computed $6,043.50 of employee FICA but then added a fabricated $64.15 “rounding adjustment.” Payroll tax is the direct sum of Social Security and Medicare components here, and no extra rounding adjustment applies." -us,scenario_033,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly computed $6,043.50 but submitted $6,045 after an unjustified whole-dollar rounding step. The requested amount is an annual dollar amount, and PolicyEngine retains the cents from Medicare tax rather than rounding the final output to $6,045." -us,scenario_033,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model stated the correct 7.65% wage-tax method but made the arithmetic wrong: 7.65% of $79,000 is $6,043.50, not $6,084. It missed the exact multiplication for the combined Social Security and Medicare rates." -us,scenario_033,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model identified the correct payroll tax base and rates but miscomputed the arithmetic. Social Security on $79,000 is $4,898 and Medicare is $1,145.50, so the total is $6,043.50 rather than $6,064." -us,scenario_033,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct Social Security and Medicare components, including $4,898 of Social Security and about $1,146 of Medicare, but then submitted $4,196, which does not follow from its own computation. It dropped or overwritten the FICA sum instead of adding the two employee payroll tax components to $6,043.50." -us,scenario_033,payroll_tax,grok-4.3,llm_error,other,False,"The model used the correct base of $79,000 in wages but rounded the FICA total to $6,045. Employee payroll tax here is $79,000 times 7.65%, which equals $6,043.50 and should not be replaced by a rounded estimate." -us,scenario_033,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable payroll_tax answer. This is a missing-output failure rather than a substantive payroll tax computation. -us,scenario_033,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly derived $4,898 of Social Security tax and $1,145.50 of Medicare tax, then submitted $9,832 after an unsupported “rough estimate considering all FICA.” The employee-side payroll tax excludes employer FICA and self-employment tax, so the correct total remains $6,043.50." -us,scenario_033,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly computed $6,043.50 from the two wage earners' 7.65% FICA liability but then rounded to $6,042. PolicyEngine's payroll_tax keeps the exact summed employee Social Security and Medicare amounts, including the $0.50 Medicare component." -us,scenario_033,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly recognized that a self-employment loss does not create self-employment tax, but then converted the loss into a negative SE tax amount. It misreported an income-loss deduction effect as the self_employment_tax output; SECA tax floors at zero when net earnings from self-employment are not positive." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly derived $4,926.40 from the $16,100 deduction and $12,400 bracket threshold, then replaced it with $4,849 based on unspecified alternative indexing. No further rounding or bracket adjustment applies after that exact calculation." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction instead of $16,100 and then miscomputed the rate schedule: income within the 12% bracket cannot produce $8,074 of tax. The correct taxable income is $43,120 and the two bracket components total $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $16,600 standard deduction rather than $16,100, calculated about $4,866 under its own assumptions, and then reduced that result to $4,297 without a tax rule. Using the enacted deduction and brackets yields $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly calculated $1,240 plus $3,686.40, or $4,926.40, and then changed the result to $4,969 under the label of rounding. Rounding cannot alter the exact bracket calculation by $42.60." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly derived taxable income of $43,120 and tax of $4,926.40, then submitted $5,194 based on unspecified bracket estimates. The stated 2026 parameters already determine the exact $4,926.40 liability." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used projected standard deductions between $15,400 and $15,750 instead of the enacted $16,100 amount. It also invented mortgage interest from the outstanding mortgage balance and assumed interest rates even though no mortgage-interest expense was listed; the correct taxable income is $43,120." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model explicitly computed the correct $4,926.40 from the $16,100 standard deduction and 2026 brackets, then submitted $6,499. The submitted value has no basis in its reasoning or in any additional tax or credit applicable to this household." -us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used the 2025 single standard deduction of $15,000 instead of the 2026 amount of $16,100. That understated the deduction by $1,100 and overstated tax by $141.50 relative to the correct $4,926.40 bracket calculation." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a post-TCJA sunset regime with a reduced standard deduction, restored personal exemption, and a 15% second bracket. For 2026 the applicable calculation uses a $16,100 standard deduction and the 10% and 12% brackets, producing $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $5,122 implies use of older deduction or bracket parameters rather than the 2026 schedule. The $16,100 standard deduction leaves $43,120 taxable, on which the 10% and 12% bracket calculation yields $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly treated the TCJA provisions as expired, restored a personal exemption, reduced the standard deduction, and applied a 15% marginal bracket. The operative 2026 rules instead provide a $16,100 standard deduction and a 12% second bracket for this income." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied an obsolete post-sunset structure consisting of an $8,300 standard deduction, a $5,050 personal exemption, and a 15% second bracket. The 2026 calculation uses the $16,100 standard deduction with 10% and 12% brackets, yielding $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly used a reduced standard deduction, a restored personal exemption, and a 15% second bracket under an assumed TCJA sunset. Applying the actual $16,100 deduction and 12% second bracket produces $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model substituted the 2024 standard deduction and 2024 bracket threshold for the requested 2026 parameters. The 2026 $16,100 deduction and $12,400 first-bracket ceiling reduce the liability to $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave no numerical derivation supporting $3,344 and that amount does not follow from the ordinary-income schedule after the standard deduction. The correct steps leave $43,120 taxable and produce $1,240 plus $3,686.40, or $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model asserted $3,720 without identifying the standard deduction or bracket thresholds used. Applying the 2026 $16,100 deduction and the 10% and 12% brackets yields $4,926.40, so its shortcut understated the ordinary-income tax." -us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated the mortgage balance as support for a mortgage-interest deduction even though no mortgage-interest payment was listed and unlisted expenses are zero. The standard deduction is $16,100, and the resulting $43,120 taxable income produces $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration, using an $8,300 standard deduction, a restored $5,300 personal exemption, and a 15% second bracket. The applicable 2026 structure uses a $16,100 standard deduction and a 12% second bracket, giving $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied a supposed pre-TCJA regime with a reduced standard deduction, restored personal exemption, and 15% second bracket. Those are not the applicable 2026 parameters; the $16,100 deduction and 10%/12% schedule yield $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model deliberately substituted the 2025 $15,000 standard deduction and 2025 brackets for the requested 2026 rules. Using the 2026 $16,100 deduction and $12,400 first-bracket ceiling produces $4,926.40 instead of $5,067.90." -us,scenario_036,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model first used an incorrect projected $15,750 deduction and derived about $4,973, then changed the submission to $6,437 without any bracket, tax, or credit supporting the increase. The enacted $16,100 deduction and 2026 rate schedule yield $4,926.40." -us,scenario_036,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model submitted a positive payroll tax amount consistent with taxing the $59,220 taxable private pension as employee payroll-tax wages. Its own explanation identified that no wages were provided and payroll tax should be zero, but the final value applied a payroll-tax base where none exists." +us,scenario_033,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly derived $6,043.50, then added a fabricated $64.15 “rounding adjustment.” The component calculations are already exact to the cent and require no adjustment." +us,scenario_033,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly calculated $6,043.50 and then improperly rounded it to $6,045. Payroll tax is requested as an annual amount, and no rule permits rounding the exact household total to a five-dollar increment." +us,scenario_033,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model stated the correct 7.65% rate and $79,000 wage base but multiplied them incorrectly. $79,000 × 0.0765 equals $6,043.50, not $6,084." +us,scenario_033,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model identified the correct wage base and FICA rates but returned an arithmetic result inconsistent with them. Social Security of $4,898 plus Medicare of $1,145.50 equals $6,043.50, not $6,064." +us,scenario_033,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model separately derived approximately $4,898 of Social Security tax and $1,146 of Medicare tax, then replaced their sum with $4,196. Using the exact Medicare amount of $1,145.50 gives total employee payroll tax of $6,043.50." +us,scenario_033,payroll_tax,grok-4.3,llm_error,other,False,"The model applied Social Security and Medicare to the correct $79,000 wage base but rounded the resulting liability incorrectly. The exact calculation is $79,000 × 7.65% = $6,043.50, not $6,045." +us,scenario_033,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required structured-output contract." +us,scenario_033,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly derived $4,898 of Social Security tax and $1,145.50 of Medicare tax, then discarded that sum and submitted an unsupported $9,832 “rough estimate.” The stated components total exactly $6,043.50, with no Additional Medicare Tax or South Dakota employee payroll tax to add." +us,scenario_033,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly calculated each spouse's employee FICA tax and the $6,043.50 total, then improperly rounded it to $6,042. No payroll-tax computation or rounding rule produces that submitted value." +us,scenario_033,payroll_tax,qwen3.8-max,llm_error,other,False,"The model identified the correct $79,000 wage base and combined 7.65% Social Security and Medicare rate but computed their product incorrectly. The product is $6,043.50, not $6,123." +us,scenario_033,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly recognized that a self-employment loss produces no self-employment tax, then contradicted that rule by reporting an invented negative liability. A negative SSTB amount cannot be multiplied by the 15.3% rate to create a self-employment tax credit; the correct liability is floored at zero." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived the exact $4,926.40 liability from the correct deduction and brackets, then replaced it with an unsupported $4,849 estimate. No rounding or bracket-indexing adjustment follows after using the applicable 2026 parameters." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction instead of $16,100 and then miscomputed the rate schedule: its stated 12% bracket arithmetic does not produce $8,074. Taxable income is $43,120, and the 10% and 12% brackets produce $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model first used an incorrect estimated $16,600 standard deduction, computed about $4,866, and then reduced that result to $4,297 without any tax provision supporting the adjustment. The applicable $16,100 deduction and 2026 brackets yield $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly derived $4,926 from $43,120 of taxable income, then changed the answer to $4,969 as an unsupported rounding adjustment. The exact bracket calculation is $4,926.40, which does not round to $4,969." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly calculated $4,926.40 using the applicable deduction and brackets, then substituted $5,194 based on unspecified parameter estimates. Once the stated 2026 parameters are applied, no further adjustment exists." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used projected standard deductions and bracket thresholds instead of the applicable $16,100 deduction and $12,400 first-bracket ceiling. It also speculated about mortgage interest from the mortgage balance even though no interest expense was listed; the prompt requires unlisted expenses to be zero." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model explicitly derived the exact $4,926.40 liability from the correct taxable income and brackets but submitted $6,499. The submitted value has no connection to its own calculation or any stated tax component." +us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used the 2025 single standard deduction of $15,000 rather than the 2026 amount of $16,100. That overstated taxable income by $1,100 and tax by $132, producing $5,067.90 instead of $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a 2026 TCJA sunset, reinstated a personal exemption, reduced the standard deduction, and applied a 15% second bracket. The applicable 2026 law uses a $16,100 standard deduction, no personal exemption, and a 12% marginal rate at $43,120 of taxable income." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $5,122 implies use of stale deduction or bracket parameters rather than the applicable 2026 schedule. The correct computation deducts $16,100 from $59,220 and applies 10% through $12,400 and 12% thereafter, yielding $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly treated the TCJA provisions as expiring for 2026, reinstating a personal exemption and applying the old 15% bracket. Applicable 2026 law instead provides a $16,100 standard deduction, no personal exemption, and a 12% bracket on the income above $12,400." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a supposed post-TCJA-sunset regime with an $8,300 standard deduction, a $5,050 personal exemption, and a 15% second bracket. The 2026 calculation instead uses the $16,100 standard deduction and 10%/12% brackets, producing $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly reinstated a personal exemption, used a reduced standard deduction, and taxed the second bracket at 15%. Under the applicable 2026 parameters, taxable income is $43,120 and remains in the 12% bracket, yielding $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model substituted the 2024 standard deduction and 2024 bracket thresholds for the requested 2026 parameters. Using the 2026 $16,100 deduction and $12,400 first-bracket ceiling reduces the liability to $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The $3,344 answer does not result from the ordinary 2026 single-filer calculation the explanation claims to apply. The pension income less the $16,100 standard deduction leaves $43,120, whose 10%/12% tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $3,720 answer understates the liability after the standard deduction and supplies no deduction, credit, or rate computation that produces it. With no nonrefundable credits, the tax on $43,120 under the 2026 10%/12% schedule remains $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model improperly invoked a mortgage-interest deduction even though only a mortgage balance was provided and unlisted interest expense must be zero. The standard deduction is $16,100, and applying the rates to the resulting $43,120 gives $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset, reinstated a personal exemption, used an $8,300 standard deduction, and applied a 15% second bracket. The applicable 2026 rules use a $16,100 standard deduction and a 12% second bracket, yielding $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a supposed pre-TCJA 2026 regime with a personal exemption, reduced standard deduction, and 15% marginal bracket. The applicable calculation has no personal exemption, deducts $16,100, and taxes the remaining $43,120 at the 10%/12% schedule." +us,scenario_036,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model deliberately substituted the 2025 $15,000 standard deduction and 2025 brackets for the requested 2026 rules. The 2026 $16,100 deduction lowers taxable income to $43,120 and the resulting tax to $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model used an incorrect $15,750 standard deduction but still computed a preliminary tax near $4,973, then changed it to $6,437 without any supporting tax step. Applying the actual $16,100 deduction and 2026 brackets yields $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model misapplied the rate schedule by placing $43,120 in a 22% bracket, invented a $3,497.60 elderly credit for a 56-year-old, and then added a medical deduction to tax rather than deducting an allowable expense from income. The entire stated sequence is invalid; the ordinary 10%/12% tax on $43,120 is $4,926.40 with no nonrefundable credits." +us,scenario_036,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly declared the $59,220 pension income below a New Jersey adult Medicaid threshold without converting the household's MAGI to its income-to-FPL level or identifying a qualifying category. The correct computation places MAGI at 3.71 times FPL and assigns medicaid_category NONE, so the head is not eligible." +us,scenario_036,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no wages were provided and that payroll tax therefore equals zero, but submitted $3,715 instead. That amount applies a payroll-tax-style rate to the $59,220 taxable private pension, which is not employee compensation subject to payroll tax." us,scenario_036,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_036,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own bracket computation produced about $1,674, but it submitted $187 with no computation connecting that number to New Jersey tax. It also improperly reduced taxable income for medical expenses instead of using the traced $58,220 base." -us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly invoked a $6,500 New Jersey standard deduction and then discarded its own $1,417.50 bracket calculation to submit $3,185. New Jersey's traced base is $58,220 after only the $1,000 personal exemption, yielding $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model correctly derived $58,220 of taxable income and $1,724.15 of tax, then wrongly deducted $8,404.60 of listed premiums and medical costs. Those costs do not reduce the traced New Jersey taxable-income base." -us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the listed health expenses and then invented an additional PolicyEngine ESI-premium reduction to reach $613. The computation uses $58,220 of taxable income with no such medical or ESI deduction." -us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted $8,404.60 as a New Jersey medical-expense deduction from the pension income. The traced taxable income subtracts only the $1,000 personal exemption, leaving $58,220 and tax of $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model granted the New Jersey retirement-income exclusion solely because income was below its stated threshold and omitted the requirement that the taxpayer be at least 62 or disabled. At age 56 with no disability, the full pension remains in New Jersey AGI." -us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied the $75,000 pension exclusion without enforcing the age-62 or disability condition. The 56-year-old therefore has $59,220 of New Jersey AGI, not zero." -us,scenario_036,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted $8,404.60 of premiums and medical expenses. The traced calculation subtracts only the $1,000 exemption from $59,220, so the taxable base is $58,220." -us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The unexplained $1,113 estimate does not follow from the New Jersey taxable base and rate schedule. Taxing $58,220 after the $1,000 exemption produces $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model applied New Jersey's pension exclusion based only on income and omitted the requirement that the filer be at least 62 or disabled. The age-56 filer cannot exclude the pension. -us,scenario_036,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used incorrect rates of 2% and 2.5% for the first two New Jersey brackets. The applicable rates are 1.4% through $20,000 and 1.75% from $20,000 through $35,000." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The submitted $2,860 does not apply the progressive New Jersey schedule to the correct $58,220 taxable base. That schedule produces $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model invoked unspecified “standard credits” to reduce the liability to $600 even though no applicable nonrefundable credit enters the traced computation. The tax on $58,220 under the New Jersey schedule remains $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted a $7,304.60 medical deduction from New Jersey income. The traced taxable base is $58,220 after only the $1,000 personal exemption." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model granted the pension exclusion based on the income threshold while omitting the age-62 or disability requirement. The age-56 filer must include all $59,220 of pension income in New Jersey AGI." -us,scenario_036,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model's unexplained $1,125 estimate omits the required derivation from taxable income and brackets. The correct computation taxes $58,220 under the New Jersey single-filer schedule and yields $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model both deducted $8,405 of medical costs and replaced New Jersey's progressive schedule with “$350 plus 2.75% over $20,000.” The traced base is $58,220, and the applicable marginal rates rise from 1.4% to 5.525% over this range." -us,scenario_036,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. -us,scenario_036,state_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model applied a pension exclusion based on income while ignoring the age-62 or disability eligibility condition and also misstated the single-filer exclusion limit. At age 56 with no disability, the pension is fully included." -us,scenario_036,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model explicitly calculated the correct $58,220 taxable income and $1,724.16 tax, then abandoned that result for an unsupported $2,283.06 “withholding-style” estimate. The requested output is the bracket-computed income-tax liability, not a withholding estimate." +us,scenario_036,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model calculated approximately $1,674 after incorrectly deducting medical expenses, then submitted $187, a value unsupported by its own computation. It also treated $2,100 of listed premiums and expenses as generating a New Jersey medical deduction, whereas this computation subtracts only the $1,000 personal exemption and taxes $58,220." +us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model imported a nonexistent New Jersey standard deduction and then abandoned its own $1,417.50 bracket calculation for an unexplained $3,185 estimate. New Jersey taxable income here is $58,220 after the $1,000 personal exemption, producing $1,724.16 under the progressive schedule." +us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model first derived the correct $58,220 taxable income and bracket tax, then incorrectly deducted $8,405 of listed insurance premiums and medical expenses. The applicable computation subtracts only the $1,000 personal exemption from the $59,220 pension." +us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $8,405 of premiums and medical expenses as deductible and then invented an additional PolicyEngine ESI-premium reduction to reach $613. PolicyEngine taxes $58,220 after only the $1,000 exemption, with no such ESI adjustment." +us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted $8,404.60 of insurance premiums and medical expenses from New Jersey income. The taxable base is $59,220 minus only the $1,000 personal exemption, so the 5.525% bracket applies to $18,220 rather than $9,815.40." +us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model ignored the age-62 eligibility condition for New Jersey's pension exclusion and treated the income thresholds as independently sufficient. At age 56 and without disability, the filer receives no pension exclusion, so the pension remains fully included." +us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied New Jersey's pension exclusion solely because income was below its limit, omitting the requirement that the taxpayer be at least 62 or otherwise qualifying. The 56-year-old filer's entire $59,220 pension is included." +us,scenario_036,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly denied the pension exclusion but incorrectly deducted $8,404.60 of listed medical costs and premiums. Only the $1,000 personal exemption reduces income here, leaving $58,220 taxable." +us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The unexplained $1,113 estimate does not result from applying New Jersey's single-filer schedule to the $58,220 taxable base. The correct bracket computation is $280 + $262.50 + $175 + 5.525% of $18,220, totaling $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model treated income below $100,000 as sufficient for New Jersey's pension exclusion and omitted its age requirement. Because the filer is 56 and not disabled, none of the $59,220 pension is excluded." +us,scenario_036,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used 2% and 2.5% for New Jersey's first two single-filer brackets instead of 1.4% and 1.75%. On $58,220 of taxable income, the correct first two bracket amounts are $280 and $262.50, yielding total tax of $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model asserted $2,860 without applying the New Jersey progressive schedule. Applying the schedule to $58,220 after the $1,000 exemption yields $1,724.16, not $2,860." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model invoked unspecified standard credits to reduce the liability to about $600, although no applicable nonrefundable New Jersey credit is present. Taxing $58,220 under the state schedule produces $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted a $7,304.60 medical deduction based on listed premiums and expenses. The computation subtracts only the $1,000 personal exemption, leaving $58,220 rather than $50,915.40 taxable." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model granted the New Jersey pension exclusion based only on total income and omitted the age-62 or qualifying-disability condition. The 56-year-old filer receives no exclusion, so the entire pension is taxable." +us,scenario_036,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The unexplained $1,125 estimate does not apply New Jersey's rate schedule to the traced taxable income. The $59,220 pension less the $1,000 exemption leaves $58,220, whose scheduled tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model both deducted $8,405 of medical expenses and replaced New Jersey's progressive brackets with a fabricated $350 plus 2.75% excess-income formula. The applicable base is $58,220 and the marginal rate from $40,000 to that amount is 5.525%." +us,scenario_036,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced taxable income by about $8,405 for listed medical expenses and premiums. Only the $1,000 exemption is subtracted, leaving $58,220 subject to New Jersey's brackets." +us,scenario_036,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no output or explanation for the requested variable. +us,scenario_036,state_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model acknowledged that the pension exclusion requires age 62, then disregarded that condition and granted the exclusion based on income alone. At age 56 and without disability, the filer cannot exclude the $59,220 pension." +us,scenario_036,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model explicitly computed the correct $1,724.16 from $58,220 of taxable income, then discarded that result and submitted an unsupported $2,283.06 withholding-style estimate. Its final value contradicts its own correct bracket calculation." +us,scenario_036,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented a $16,100 New Jersey standard deduction and then added a purported property-tax deduction to tax, reversing the function of a deduction and using an unlisted expense. New Jersey taxable income here is $58,220 after the $1,000 personal exemption only." us,scenario_036,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_037,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model denied the qualified-overtime deduction and incorrectly deducted $5,789 of employer-sponsored insurance premiums from the stated gross wages. Its submitted $1,646 also contradicts its own displayed $2,308 calculation." -us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model stopped after the $16,100 standard deduction and omitted the additional $9,629.03 qualified-overtime deduction. That omission left taxable income near $25,439 instead of $15,810.43." -us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted the $9,629.03 qualified-overtime deduction and then submitted $3,160 despite its own bracket calculation producing about $2,805. The correct deductions leave $15,810.43 of taxable income." -us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model recognized the qualified-overtime deduction but did not actually subtract the full $9,629.03 in its tax computation. Full application alongside the $16,100 standard deduction reduces taxable income to $15,810.43, not an amount producing $2,570 of tax." -us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated qualified overtime as an exclusion from gross income instead of an additional deduction and used guessed standard-deduction and bracket values. It also submitted $1,569 despite repeatedly calculating values around $1,700; the traced deduction method yields $15,810.43 of taxable income." -us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $9,629.03 qualified-overtime deduction after computing AGI. Its $1,808 submission also does not follow from its displayed $2,809.68 calculation." -us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed a TCJA sunset, used a reduced standard deduction plus personal exemption, deducted the separately listed ESI premium from wages, and omitted the qualified-overtime deduction. For 2026 the applicable deductions are the $16,100 standard deduction and $9,629.03 overtime deduction." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied a TCJA-sunset standard deduction, personal exemption, and 15% bracket while also subtracting the ESI premium from gross wages. It omitted the $9,629.03 qualified-overtime deduction that brings taxable income to $15,810.43." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The explanation omits both the $18.03 above-the-line deduction and the $9,629.03 qualified-overtime deduction and uses the wrong standard deduction. Its $1,109 answer does not follow from the taxable income implied by its stated inputs." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the ESI premium, assumed the TCJA sunset, and omitted the qualified-overtime deduction. It also invented a $50 Saver's Credit instead of applying the traced deductions that yield $15,810.43 of taxable income." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly used post-sunset deductions, a personal exemption, and a 15% bracket, while subtracting ESI premiums from gross wages. It never applied the $9,629.03 qualified-overtime deduction." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $5,789 of ESI premiums from the stated wages and omitted the $9,629.03 qualified-overtime deduction. It also used a $15,450 standard deduction instead of $16,100." -us,scenario_037,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an outdated $14,600 standard deduction and entirely omitted the $9,629.03 qualified-overtime deduction. Those errors inflated taxable income from $15,810.43 to $26,939." -us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction and unspecified credits as sufficient to erase all liability. After the $16,100 standard deduction and $9,629.03 overtime deduction, taxable income remains $15,810.43 and produces $1,649.25 of tax." -us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invented unspecified nonrefundable credits and incorrectly concluded that deductions eliminate taxable income. The actual deductions leave $15,810.43 taxable, with no credits reducing the resulting $1,649.25 liability." -us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model omitted both the $385.90 pre-tax 401(k) reduction and the $9,629.03 qualified-overtime deduction from its taxable-income calculation. It consequently used $26,175 rather than $15,810.43 of taxable income." -us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its answer is consistent with subtracting retirement adjustments and a standard deduction while omitting the $9,629.03 qualified-overtime deduction. Applying that additional deduction leaves $15,810.43 of taxable income and $1,649.25 of tax." -us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed a TCJA sunset, substituted a reduced standard deduction and personal exemption, and used a 15% bracket. It also omitted the $9,629.03 qualified-overtime deduction." -us,scenario_037,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the ESI premium, assumed reverted deductions and brackets, and omitted the qualified-overtime deduction. The applicable calculation instead combines a $16,100 standard deduction with the $9,629.03 overtime deduction." -us,scenario_037,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response failed the required output contract." -us,scenario_037,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that $41,557 of wages after the 401(k) contribution falls below the standard deduction and further invoked a personal exemption. The valid deductions still leave $15,810.43 of taxable income rather than zero." -us,scenario_037,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $18.03 above-the-line IRA deduction and the entire $9,629.03 qualified-overtime deduction, while using a $15,600 rather than $16,100 standard deduction. This inflated taxable income to $25,957 instead of $15,810.43." -us,scenario_037,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_037,head_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model correctly identified North Carolina expansion but applied it without the MAGI income limit for the adult expansion group. At 2.60 x FPL and with no SSI, dependent, disability, pregnancy, child, aged, or other category pathway, the head qualifies through none of the Medicaid pathways, so Medicaid eligibility is 0." -us,scenario_037,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model's own component calculation produced $3,208.74, but it then replaced that result with $3,211.64 based on an unexplained “minor adjustment.” The traced components are $2,600.49 and $608.18, which sum to $3,208.67." -us,scenario_037,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $5,789 employer-sponsored insurance premium from FICA wages. That amount is not specified as an employee pre-tax payroll deduction, so Social Security and Medicare taxes apply to the full wages and total $3,208.67." -us,scenario_037,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated the listed employer-sponsored insurance premium as a pre-tax employee contribution and reduced FICA wages from $41,943 to $36,154. PolicyEngine taxes the full wage base here, producing $3,208.67." -us,scenario_037,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly excluded the $5,789 employer-sponsored insurance premium from Social Security and Medicare wages. The prompt does not identify it as a pre-tax payroll deduction, and the full wage base yields $3,208.67." -us,scenario_037,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model reduced FICA wages by the employer-sponsored insurance premium despite no stated employee pre-tax payroll deduction. Applying the traced Social Security and Medicare calculations to full wages yields $3,208.67." -us,scenario_037,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly estimated the federal components at about $3,209 and correctly excluded an NC employee payroll tax, but then submitted $3,151 without a supporting computation. The exact traced components total $3,208.67." -us,scenario_037,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model rounded the employee Social Security and Medicare liability to $3,210 instead of calculating the requested annual amount to cents. The traced components of $2,600.49 and $608.18 total $3,208.67." -us,scenario_037,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly treated the $5,789 employer-sponsored insurance premium as a pre-tax employee deduction from FICA wages. It also rounded each component to whole dollars; using the full wage base and traced component amounts gives $3,208.67." -us,scenario_037,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll-tax output or explanation, violating the required output contract." -us,scenario_037,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model used coarse approximations for both FICA components and then overstated their approximate sum as $3,213. The exact traced Social Security tax of $2,600.49 plus Medicare tax of $608.18 equals $3,208.67." -us,scenario_037,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $10,750 standard deduction and then discarded its own positive tax calculation by asserting that unspecified deductions and credits reduced liability to zero. The household has no applicable nonrefundable North Carolina credit, so $28,789.45 of taxable income produces $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the stated gross wages as already reduced by the traditional retirement contributions and used $41,943 instead of $41,539.45 as AGI. Subtracting the $12,750 standard deduction from the correct AGI yields $28,789.45 and $1,148.70 of tax." -us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model derived essentially the correct taxable income and an approximately correct tax, then replaced that result with an unsupported $1,180 adjustment. No contribution-related adjustment remains after deriving AGI and taxable income, so the schedule yields $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived approximately $28,789 of taxable income and $1,149 of tax, then invented an unspecified decoupling or deduction adjustment that reduced the answer to $883. No such adjustment applies to this household; the computed liability remains $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model cycled through several contradictory standard deductions and ultimately used an invented $21,500 single deduction to support $803. The applicable deduction is $12,750, leaving $28,789.45 taxable and producing $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly calculated $28,789 of taxable income but first applied the wrong 4.25% rate and then replaced even that result with an unsupported $1,685 estimate. Applying the 2026 schedule to $28,789.45 yields $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $35,750 instead of $41,539.45, thereby reducing taxable income from $28,789.45 to $23,000. The unsupported $5,789 reduction corresponds to treating the separately listed employer-sponsored insurance premium as an additional deduction from wages." -us,scenario_037,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $35,750 and also used an incorrect $12,850 state standard deduction. The correct inputs are $41,539.45 of AGI and a $12,750 deduction, producing $28,789.45 of taxable income and $1,148.70 of tax." -us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $1,787.52 is consistent with applying an approximate 4.3% rate directly to nearly all wage income without first subtracting the North Carolina standard deduction. Tax must be computed on $28,789.45 after the $12,750 deduction under the 2026 schedule, yielding $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $35,750 instead of $41,539.45, leaving only $23,000 taxable after the standard deduction. Using the correct AGI leaves $28,789.45 taxable and produces $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated North Carolina AGI as $35,750 and therefore calculated tax on only $23,000. AGI is $41,539.45; after the $12,750 standard deduction, the $28,789.45 taxable base produces $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $35,750 starting AGI, reducing taxable income to $23,000. The correct starting AGI is $41,539.45, so taxable income is $28,789.45 and tax is $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an incorrect estimated single standard deduction of $15,500 rather than $12,750. That overstated deductions by $2,750 and reduced the taxable base incorrectly; the correct $28,789.45 base yields $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model supplied no supporting arithmetic, and $1,268 does not result from applying the 2026 North Carolina schedule to the correct taxable income. AGI of $41,539.45 less the $12,750 standard deduction gives $28,789.45 and tax of $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that deductions and credits eliminate the state tax. The $12,750 standard deduction leaves $28,789.45 taxable, and no applicable nonrefundable credit reduces the resulting $1,148.70 liability to zero." -us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model's own arithmetic produced $1,148.68 but it submitted $1,147, so the final value did not follow its computation. Applying the engine's precise AGI and schedule produces $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model understated taxable income as $19,160 by subtracting roughly $9,629 of unsupported additional deductions, matching the separately listed FLSA overtime premium. That premium is part of annual gross wages, not an above-the-line deduction; taxable income is $28,789.45 and tax is $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave only an unsupported $1,370 flat-tax result and did not apply the specified 2026 North Carolina computation. Applying the state schedule to $28,789.45 of taxable income yields $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $35,750 and rounded the applicable rate to 4%, producing tax on only $23,000. The correct taxable income is $28,789.45 after the $12,750 deduction, and the 2026 schedule yields $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model deducted the $5,789 employer-sponsored insurance premium from gross wages even though PolicyEngine's traced AGI is $41,539.45 after the applicable retirement deductions. Removing that extra deduction leaves $28,789.45 taxable after the state standard deduction and produces $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly claimed that pre-tax adjustments and the standard deduction reduce taxable income to nearly zero and also used the wrong 4.5% rate. The applicable deductions leave $28,789.45 taxable, producing $1,148.70 rather than zero." -us,scenario_037,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used the wrong 4.5% rate and omitted the deductible traditional IRA contribution from AGI, then submitted $1,773.56 despite repeatedly calculating about $1,285 to $1,296. The correct AGI is $41,539.45, the taxable base is $28,789.45, and the 2026 schedule produces $1,148.70." -us,scenario_038,child1_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model checked only CHIP's age and upper-income conditions and omitted the prerequisite that the child not qualify for Medicaid. Child 1 qualifies for Louisiana Medicaid under the OLDER_CHILD category, so CHIP eligibility is 0 regardless of being below the CHIP income ceiling." -us,scenario_038,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model treated income below roughly 250% FPL as sufficient for CHIP and skipped the Medicaid-eligibility exclusion. The child's low household income instead establishes Medicaid eligibility under Louisiana's OLDER_CHILD category, which disqualifies the child from CHIP." -us,scenario_038,child1_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model applied the CHIP age and maximum-income tests without first determining Medicaid eligibility. Child 1 is Medicaid-eligible under Louisiana's OLDER_CHILD pathway, and CHIP covers children who do not qualify for Medicaid." -us,scenario_038,child1_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model concluded that being uninsured, under 19, and below 250% FPL establishes CHIP eligibility, omitting the Medicaid exclusion. At this income, Child 1 qualifies for Medicaid as an OLDER_CHILD, so the child is not eligible for CHIP." -us,scenario_038,child1_chip_eligible,claude-sonnet-5,llm_error,health_coverage,False,The model conflated the combined LaCHIP/Medicaid coverage system with eligibility for the specific CHIP output. Its own statement that the child qualifies for Medicaid identifies the disqualifying condition: Medicaid-eligible children are not CHIP-eligible. -us,scenario_038,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for child1_chip_eligible. It therefore failed the required output contract rather than completing the Medicaid-first eligibility determination. -us,scenario_038,child1_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model merged Louisiana's CHIP and Medicaid thresholds into a single child-coverage test and returned CHIP eligibility from low income alone. The approximately 73% FPL income makes Child 1 Medicaid-eligible under the OLDER_CHILD category, and that eligibility excludes the child from CHIP." -us,scenario_038,child1_head_start_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model acknowledged that age 7 is above the typical Head Start preschool range but then ignored that categorical age constraint for this benchmark output. It also used an incorrect 2026 family-of-four poverty comparison, treating about $22,993 as below the relevant cutoff that PolicyEngine applies for Louisiana Head Start eligibility." -us,scenario_038,child1_head_start_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model stated that Head Start serves ages 3-5 but then awarded eligibility to a 7-year-old by treating income eligibility as sufficient. It also compared the household's roughly $22,993 income to the wrong poverty guideline threshold instead of applying PolicyEngine's Louisiana Head Start cutoff used in the trace." -us,scenario_038,child1_head_start_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,The model incorrectly classified a 7-year-old as within the preschool-age range for Head Start. It then relied on a vague low-income shortcut from wages plus self-employment income rather than applying the PolicyEngine Head Start age category and Louisiana income cutoff that make Child 1 ineligible. -us,scenario_038,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the older child Medicaid pathway for a 7-year-old dependent in Louisiana and reduced the question to an unsupported income/assets expectation. PolicyEngine's MAGI calculation is 0.66 times FPL, which satisfies the Louisiana 2026 older-child Medicaid income limit, so child1 is Medicaid eligible." -us,scenario_038,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model applied the wrong WIC child age rule by treating a 7-year-old as within the WIC child category. WIC child eligibility ends before age 5, so Child 1 fails categorical eligibility before any income pathway can make the child eligible." -us,scenario_038,child2_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied CHIP age and income tests directly and treated low assets as relevant, but never checked the prerequisite that the child be ineligible for Medicaid. Child 2 qualifies for Medicaid under the OLDER_CHILD category, which excludes the child from CHIP regardless of being below the cited CHIP income limit." -us,scenario_038,child2_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model inferred CHIP eligibility solely from age and household income below its stated CHIP limit. It omitted the Medicaid-first eligibility step: Child 2 qualifies for Medicaid under the OLDER_CHILD category and therefore cannot qualify for CHIP. -us,scenario_038,child2_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated income within CHIP limits as sufficient for coverage. It failed to determine that Child 2 is Medicaid-eligible under the OLDER_CHILD category, and that existing Medicaid eligibility makes the child ineligible for CHIP." -us,scenario_038,child2_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model checked age, lack of listed insurance, and the CHIP income ceiling but omitted the controlling Medicaid-exclusion rule. Child 2 qualifies for Medicaid under the OLDER_CHILD category, so being uninsured and below the CHIP ceiling does not establish CHIP eligibility." -us,scenario_038,child2_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model concluded that low countable income and the absence of an asset test establish LaCHIP eligibility, without first checking Medicaid eligibility. Child 2 qualifies for Medicaid under the OLDER_CHILD category, and that qualification bars CHIP eligibility." -us,scenario_038,child2_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,The model used only the under-19 and CHIP income-limit conditions. It skipped the required Medicaid screen: Child 2 qualifies for Medicaid under the OLDER_CHILD category and is consequently excluded from CHIP. -us,scenario_038,child2_head_start_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a 5-person poverty threshold even though the prompt lists four household members, inflating the Head Start income limit to $29,960. Under the PolicyEngine Head Start test, the family unit is four people and roughly $22,992 of income exceeds the applicable threshold, so child2 is not income-eligible." -us,scenario_038,child2_head_start_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model asserted that roughly $23,000 for a family of four is below the Head Start poverty limit and then overrode its own age concern with an unsupported age-proximity assumption. PolicyEngine applies the Head Start income threshold to the four-person household and the counted income exceeds that threshold, producing no eligibility for child2." -us,scenario_038,child2_head_start_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model treated the household as low-income for Head Start and used a discretionary kindergarten-transition rationale to submit eligibility despite recognizing that age 6 is outside the usual preschool range. PolicyEngine's computation fails the case on the income screen because counted income of about $22,992 exceeds the applicable Head Start threshold for the four-person family." -us,scenario_038,child2_head_start_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used an overstated 2026 Head Start poverty guideline of about $32,150 for a family of four, making the household appear comfortably income-eligible. PolicyEngine uses the applicable Head Start income threshold for the Louisiana four-person household, and the household's roughly $22,992 income exceeds that threshold, so child2 is not eligible." -us,scenario_038,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply Louisiana's older child Medicaid category for a 6-year-old tax-unit dependent and instead dismissed eligibility based on an unsupported general income/assets screen. Under the MAGI child pathway, child2's household income is 0.66 times FPL, below the applicable older-child limit, so the eligibility indicator is 1." -us,scenario_038,child2_wic_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an incorrect 2026 WIC income threshold for a four-person Louisiana household, treating 185% FPL as roughly $52,000 and therefore classifying $22,993 as under the limit. It also incorrectly treated a 6-year-old as within the WIC child age category, despite WIC child eligibility ending before age 6." -us,scenario_038,child2_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model misapplied the WIC child age rule by counting a 6-year-old as covered through the age-5 child category. It also relied on a generic low-income comparison and failed to apply the PolicyEngine WIC income limit that makes this household ineligible. -us,scenario_038,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model's answer rests entirely on the false premise that age 6 is within the WIC child age range used for this benchmark. Under the PolicyEngine computation, child 2 is not WIC eligible; its shortcut never applied the household income screen that also returns ineligibility." -us,scenario_038,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model did not provide a parseable value for federal_income_tax_before_refundable_credits, so it failed the required output contract rather than making a substantive tax computation." -us,scenario_038,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own reasoning correctly computed zero taxable income, zero tax before credits, and zero usable nonrefundable credits, which yields federal_income_tax_before_refundable_credits = 0. It then submitted -1644, a value inconsistent with the definition because this pre-refundable-credits tax cannot go below zero and does not subtract refundable credits." -us,scenario_038,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model abandoned its own near-correct refundable CTC calculation and substituted an unsupported lower EITC estimate. The applicable components are EITC of $7,316 and refundable CTC of $2,909.35, not the model's ad hoc $10,148 total." -us,scenario_038,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model used an incorrect EITC phase-in ceiling and then imposed a $1,656.20 phaseout even though this joint household receives the full $7,316 two-child EITC. It also granted the full $3,400 refundable CTC cap instead of applying the earned-income limit that yields $2,909.35." -us,scenario_038,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model understated both components despite computing the refundable CTC earned-income limit as about $2,909. It replaced that result with $2,383 and used an approximate $7,150 EITC instead of $7,316." -us,scenario_038,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model reduced the two-child EITC to roughly $5,500 without applying the 2026 plateau amount of $7,316. It also cut refundable CTC to $1,527 even though the earned-income phase-in produces $2,909.35." -us,scenario_038,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model correctly recognized that refundable CTC is governed by the 15% earned-income formula but did not carry its own roughly $3,074 calculation into the total. It also replaced the exact $7,316 EITC with an unspecified lower estimate, producing $9,199." -us,scenario_038,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model repeatedly substituted obsolete or projected 2025 EITC parameters for the applicable 2026 amount and ultimately used $6,604 instead of $7,316. Its refundable CTC calculation was essentially $2,909, so the remaining shortfall comes from the wrong EITC maximum." -us,scenario_038,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model first derived components near $6,900 and $2,896, then discarded them for an unsupported $8,500 total. The computation requires the exact $7,316 EITC and $2,909.35 refundable CTC." -us,scenario_038,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied obsolete refundable CTC rules: a $1,000-per-child cap and a $3,000 earned-income threshold. Under the applicable rules, the refundable CTC is $2,909.35, which combines with the $7,316 EITC." -us,scenario_038,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model limited refundable CTC to $1,000 per child and used a $3,000 phase-in threshold, yielding only $2,000. The applicable earned-income phase-in yields $2,909.35, and the exact EITC is $7,316 rather than an estimated $7,300." -us,scenario_038,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model treated the full $4,000 nominal CTC as refundable and simultaneously understated the two-child EITC as $4,100. Only $2,909.35 of CTC is refundable under the earned-income limitation, while the EITC is $7,316." -us,scenario_038,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assigned only $2,000 to refundable CTC, consistent with an obsolete $1,000-per-child rule, and rounded the EITC to $7,300. The applicable amounts are $2,909.35 and $7,316." -us,scenario_038,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an obsolete $1,000-per-child refundable CTC cap, limiting that component to $2,000 instead of $2,909.35. It also approximated the EITC at $7,250 rather than applying the $7,316 value." -us,scenario_038,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The answer identifies the two relevant credits but does not compute either one and understates their sum. The required calculation is $7,316 of EITC plus $2,909.35 of refundable CTC, totaling $10,225.35." -us,scenario_038,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the submission contract." -us,scenario_038,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model named EITC and refundable CTC but provided no component calculation and understated their combined value. Those components are $7,316 and $2,909.35, not $8,724 in total." -us,scenario_038,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that low earnings prevent refundable credits. A joint filer with two qualifying children and this earned income receives both a $7,316 EITC and $2,909.35 of refundable CTC even with zero regular income-tax liability." -us,scenario_038,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model correctly calculated the $7,316 EITC but automatically granted $1,800 per child as refundable CTC. The refundable portion is constrained by the earned-income phase-in to $2,909.35, not $3,600." -us,scenario_038,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used $8,231 as the two-child EITC, exceeding the applicable $7,316 maximum, and calculated refundable CTC from gross earned income as $3,073.95. The applicable refundable CTC computation yields $2,909.35." -us,scenario_038,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model got the $7,316 EITC exactly right but understated refundable CTC as $2,731.32. Applying the earned-income phase-in produces $2,909.35." -us,scenario_038,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model treated the instruction against inferring unlisted facts as barring credits generated by the listed earnings and children. Those supplied facts directly establish a $7,316 EITC and $2,909.35 refundable CTC; neither requires positive pre-credit income tax." -us,scenario_038,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a $2,000 aggregate ACTC cap, effectively using an obsolete $1,000-per-child refundable limit. The earned-income calculation instead permits $2,909.35 of refundable CTC, and the exact EITC is $7,316 rather than $7,312." -us,scenario_038,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used post-TCJA-reversion assumptions—a $1,000-per-child refundable CTC and a $3,000 threshold—that do not govern this calculation. Refundable CTC is $2,909.35 under the applicable phase-in, and EITC is $7,316 rather than $7,338." -us,scenario_038,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the submission contract." -us,scenario_038,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly used the $7,316 EITC but based refundable CTC on gross wages plus gross self-employment income, producing $3,073.95. The applicable earned-income computation accounts for the deductible half of self-employment tax and yields $2,909.35." -us,scenario_038,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model understated EITC as roughly $6,500 and automatically granted the full $3,400 refundable CTC cap. The household receives a $7,316 EITC, while the CTC earned-income phase-in limits its refundable portion to $2,909.35." -us,scenario_038,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model mixed three-child and two-child EITC maxima, used the wrong joint phaseout threshold, alternated between net and gross self-employment earnings for refundable CTC, and finally discarded its own calculations. Applying the 2026 rules consistently gives a $7,316 EITC and $2,909.35 refundable CTC." -us,scenario_038,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model declared the household ineligible without applying either qualifying pathway. The household is categorically eligible through $7,286.94 of computed SNAP benefits and independently passes the free-meal income test at 70% of the federal poverty guideline." -us,scenario_038,free_school_meals_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model incorrectly treated unlisted SNAP receipt as zero even though SNAP was a requested computed output, not an input that had to be explicitly listed; PolicyEngine computes $7,286.94 of SNAP, which confers categorical eligibility. It also acknowledged gross income below 130% of the guideline but then wrongly imposed a categorical-link or actual-receipt requirement, overlooking the independent income-certification route and the benchmark's definition based on positive annual support." +us,scenario_037,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model denied the 2026 qualified-overtime deduction and used a $14,600 standard deduction instead of $16,100. It also submitted $1,646 despite its own displayed computation producing $2,308, whereas the overtime deduction leaves $15,810.43 taxable income and $1,649.25 of tax." +us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model correctly reached approximately $41,539 of AGI and used the $16,100 standard deduction, but omitted the additional $9,629.03 qualified-overtime deduction. It then submitted $2,884 despite its own bracket calculation yielding about $2,805." +us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted the $9,629.03 qualified-overtime deduction and therefore taxed about $25,439 instead of $15,810.43. Its final $3,160 also contradicts its own calculated tax of approximately $2,805." +us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model recognized the qualified-overtime deduction but did not subtract its full $9,629 amount: applying it to the model's $25,439 pre-overtime taxable income leaves about $15,810, not an amount producing $2,570 of tax. The correct bracket calculation on $15,810.43 is $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated qualified overtime as an exclusion from gross income rather than as a deduction and used estimated standard deductions and brackets instead of the 2026 values. Even its final $1,569 contradicts every displayed recomputation, including its last calculation of $1,700; the proper $16,100 standard deduction plus $9,629.03 overtime deduction yields $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the entire $9,629.03 qualified-overtime deduction after correctly approximating AGI and the $16,100 standard deduction. Its submitted $1,808 also does not follow from its stated $2,809.68 bracket calculation." +us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset for 2026, replacing the applicable $16,100 standard deduction and 10%/12% schedule with a small standard deduction, personal exemption, and 15% bracket. It also omitted the $9,629.03 qualified-overtime deduction and improperly subtracted the separately listed ESI premium from wages." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA personal exemption, reduced standard deduction, and 15% bracket rather than the applicable 2026 $16,100 standard deduction and rate schedule. It also omitted the $9,629.03 qualified-overtime deduction and double-counted the ESI premium as an AGI reduction." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's sparse derivation omitted both the $18.03 above-the-line IRA deduction and the $9,629.03 qualified-overtime deduction, while using a $15,800 standard deduction instead of $16,100. Its $1,109 result does not follow from the taxable income implied by its stated AGI and deduction." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA sunset and used a personal exemption and reduced standard deduction, while also subtracting the ESI premium from wages and omitting the $9,629.03 qualified-overtime deduction. It additionally invented a $50 Saver's Credit rather than computing the traced liability from $15,810.43 of taxable income." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly reverted to a personal exemption, reduced standard deduction, and 15% bracket for 2026. It also double-counted the listed ESI premium as an AGI reduction and omitted the $9,629.03 qualified-overtime deduction." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $5,789 of ESI premiums from the stated wage input and omitted the $9,629.03 qualified-overtime deduction. It also used a $15,450 standard deduction instead of $16,100, producing $20,300 of taxable income rather than $15,810.43." +us,scenario_037,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the $9,629.03 qualified-overtime deduction and used a $14,600 standard deduction instead of $16,100. Those errors made it tax $26,939 rather than the traced $15,810.43." +us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction and unspecified credits as sufficient to erase the liability, but the standard deduction is not a credit and no nonrefundable credit offsets the tax here. After all deductions, $15,810.43 remains taxable and produces $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the standard deduction as a credit and invented unspecified education or retirement credits that the prompt sets to zero absent supporting facts. The actual deductions leave $15,810.43 taxable, with no nonrefundable credits reducing the resulting $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $385.90 pre-tax 401(k) reduction and the full $9,629.03 qualified-overtime deduction, and used a $15,750 standard deduction instead of $16,100. It therefore taxed $26,175 rather than $15,810.43." +us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The submitted amount is consistent with applying retirement adjustments and a standard deduction while omitting the additional $9,629.03 qualified-overtime deduction. Including that deduction leaves $15,810.43 taxable and produces $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly used a reduced post-sunset standard deduction, a personal exemption, and a 15% bracket for 2026. It also omitted the $9,629.03 qualified-overtime deduction, so its $27,939 taxable-income figure is $12,128.57 above the traced amount." +us,scenario_037,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly applied reverted brackets, a reduced standard deduction, and a personal exemption for 2026. It also subtracted the ESI premium from wages and omitted the $9,629.03 qualified-overtime deduction." +us,scenario_037,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model improperly reduced AGI by the separately listed ESI premium and omitted the $9,629.03 qualified-overtime deduction. It also invented a $40.40 Saver's Credit; the traced calculation instead uses $41,539.45 of AGI, $25,729.03 of deductions, and no credit adjustment." +us,scenario_037,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so it failed the required output contract." +us,scenario_037,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that wages after the 401(k) contribution fall below the standard deduction and added a personal exemption that does not apply. Even after the standard and qualified-overtime deductions, taxable income is $15,810.43 rather than zero." +us,scenario_037,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted both the $18.03 above-the-line IRA deduction and the $9,629.03 qualified-overtime deduction, and used a $15,600 standard deduction instead of $16,100. It consequently taxed $25,957 rather than $15,810.43." +us,scenario_037,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a Child Tax Credit for a household containing no child or other dependent. No CTC applies, and the deductions leave a $1,649.25 liability before refundable credits rather than zero." +us,scenario_037,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable output for federal_refundable_credits and therefore failed the required submission contract. +us,scenario_037,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented dependent-child credit eligibility despite the household containing only one 44-year-old adult: refundable CTC is $0 because there is no qualifying child. It also applied a child-based EITC amount even though the filer has no qualifying children, and $41,943 exceeds the childless EITC income limit, so EITC is $0." +us,scenario_037,head_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model treated North Carolina’s adoption of Medicaid expansion as sufficient for eligibility and failed to compare the head’s MAGI income of 2.60 times FPL with the expansion-adult income limit. The head exceeds that limit and has no alternative Medicaid pathway, yielding medicaid_category NONE and ineligibility." +us,scenario_037,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model's stated component calculation produced $3,208.74, but it then submitted $3,211.64 after inventing an unsupported “minor adjustment.” It failed to use the traced Social Security and Medicare components of $2,600.49 and $608.18, which sum to $3,208.67." +us,scenario_037,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $5,789 employer-sponsored insurance premium from wages even though the facts do not identify it as an employee pre-tax payroll deduction. Applying employee Social Security and Medicare taxes to the full traced wage base yields $3,208.67." +us,scenario_037,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated the listed employer-sponsored insurance premium as a pre-tax employee contribution and reduced FICA wages from $41,943 to $36,154. That unsupported deduction excluded wages subject to Social Security and Medicare tax and produced the $2,765.78 answer." +us,scenario_037,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly excluded the entire $5,789 employer-sponsored insurance premium from the payroll-tax base. The input establishes ESI coverage and a premium amount, not an employee pre-tax payroll deduction, so the traced taxes are $2,600.49 for Social Security and $608.18 for Medicare." +us,scenario_037,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model reduced taxable wages by $5,789 based solely on the employer-sponsored insurance premium field. Because that amount is not specified as a pre-tax employee payroll contribution, its $36,154 FICA base is wrong and omits $442.89 of traced payroll tax." +us,scenario_037,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly estimated that the Social Security and Medicare components total about $3,209, then submitted $3,151 without any supporting computation. The traced components sum exactly to $3,208.67, and no North Carolina employee payroll tax changes that total." +us,scenario_037,payroll_tax,grok-4.3,llm_error,other,False,"The model rounded an unspecified Social Security and Medicare calculation to $3,210 instead of calculating the traced components. Employee Social Security tax of $2,600.49 plus Medicare tax of $608.18 equals $3,208.67." +us,scenario_037,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly treated the $5,789 employer-sponsored insurance premium as a pre-tax employee deduction and calculated FICA on only $36,154. The premium field does not authorize that payroll-base reduction, so Social Security and Medicare apply to the full traced wage base." +us,scenario_037,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model excluded $5,789 from FICA wages by assuming the employer-sponsored insurance premium was paid through a pre-tax employee payroll deduction. The stated facts do not make that deduction, and the traced full-base Social Security and Medicare liabilities total $3,208.67." +us,scenario_037,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract rather than completing a substantive calculation." +us,scenario_037,payroll_tax,minimax-m3,llm_error,other,False,"The model used coarse approximations for both payroll-tax components and then overstated their approximate sum as $3,213. The traced Social Security amount is $2,600.49 and the traced Medicare amount is $608.18, totaling $3,208.67." +us,scenario_037,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model correctly computed a positive liability of about $1,386 under its own assumptions, then replaced it with zero without identifying any applicable credit or deduction. This household has $28,789.45 of North Carolina taxable income and $1,148.70 of tax before refundable credits." +us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model treated the stated gross wages as already reduced by the traditional contributions and therefore failed to subtract the $386 traditional 401(k) and $18 traditional IRA contributions. Adjusted gross income is $41,539.45, not $41,943, producing $28,789.45 of taxable income and $1,148.70 of tax." +us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model derived taxable income of about $28,789 and an initial tax near the correct result, then arbitrarily increased the answer to $1,180 for unspecified contribution effects. The traditional contributions were already reflected in its AGI calculation, so no further adjustment applies." +us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived approximately $1,149 and then reduced it to $883 through an unsupported decoupling adjustment. No North Carolina add-back, deduction, or credit in the household facts produces that reduction." +us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model ultimately invented a $21,500 single standard deduction to force the liability down to $803, despite also deriving $1,148.68 using the applicable $12,750 deduction. The correct taxable income is $28,789.45, not roughly $20,057." +us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a 4.25% rate instead of the applicable 2026 North Carolina rate schedule and then raised its own computed $1,223.50 to $1,685 without a supporting calculation. Applying the proper parameters to $28,789.45 yields $1,148.70." +us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated adjusted gross income as $35,750, which reflects an extra $5,789 reduction beyond the traditional contributions. The employer-sponsored insurance premium must not be subtracted from the supplied wages in this calculation; adjusted gross income is $41,539.45." +us,scenario_037,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $35,750 adjusted gross income and also substituted a $12,850 standard deduction for North Carolina's $12,750 amount. The correct taxable income is $28,789.45 rather than $22,900." +us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $1,787.52 applies an approximate 4.3% rate to nearly all wage income instead of first subtracting the traditional contributions and North Carolina's $12,750 standard deduction. Only $28,789.45 is taxable under the state calculation." +us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated adjusted gross income as $35,750 and consequently taxed only $23,000 after the standard deduction. Adjusted gross income is $41,539.45 and taxable income is $28,789.45." +us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used $35,750 as state adjusted gross income, improperly removing an additional $5,789 from income. With adjusted gross income of $41,539.45 and the $12,750 deduction, North Carolina taxes $28,789.45." +us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model began from an unsupported adjusted gross income of $35,750 rather than $41,539.45. That error reduced taxable income from $28,789.45 to $23,000 and understated the tax." +us,scenario_037,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented a $15,500 North Carolina single standard deduction instead of using $12,750. This understated taxable income by $2,750 and produced $1,039 rather than $1,148.70." +us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The unexplained $1,268 answer does not follow the required calculation: $41,539.45 of adjusted gross income less the $12,750 standard deduction leaves $28,789.45, whose tax is $1,148.70. Its answer therefore embeds an incorrect taxable base, rate, or both." +us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that deductions and credits eliminate the tax. The standard deduction leaves $28,789.45 of taxable income, and no applicable nonrefundable credit reduces the resulting $1,148.70 liability to zero." +us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model's own stated inputs and multiplication produced $1,148.68, but it submitted $1,147. The engine's precise taxable income and rate-schedule calculation yield $1,148.70, so the error is an unsupported final-value substitution rather than a policy adjustment." +us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to $19,160 without identifying deductions that bridge from adjusted gross income to that amount. North Carolina's $12,750 standard deduction leaves $28,789.45 taxable, not $19,160." +us,scenario_037,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model's unexplained $1,370 does not result from applying the 2026 North Carolina calculation to $28,789.45 of taxable income. The applicable rate schedule produces $1,148.70." +us,scenario_037,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used $35,750 of adjusted gross income and taxed only $23,000 after the standard deduction. The correct adjusted gross income is $41,539.45 and the correct taxable income is $28,789.45; it also rounded the 3.99% rate to 4%." +us,scenario_037,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model understated adjusted gross income as approximately $35,750, implicitly subtracting an unsupported additional amount from wages. Adjusted gross income is $41,539.45, leaving $28,789.45 after the state standard deduction." +us,scenario_037,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model separately subtracted the $5,789 employer-sponsored insurance premium from the supplied gross wages, reducing adjusted gross income to $35,750. That premium is not an additional deduction from the benchmark wage input, so adjusted gross income remains $41,539.45 after the traditional contributions." +us,scenario_037,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that pre-tax adjustments and the standard deduction reduce taxable income to nearly zero. They leave $28,789.45 of North Carolina taxable income, and the model also used the wrong 4.5% rate." +us,scenario_037,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model repeatedly used a 4.5% rate instead of the applicable 2026 North Carolina rate schedule, yet its submitted $1,773.56 also contradicted every intermediate result it calculated. Using adjusted gross income of $41,539.45 and the $12,750 deduction yields $1,148.70." +us,scenario_037,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model supplied no computation supporting $1,234. The required 2026 calculation taxes $28,789.45 after the $12,750 standard deduction and yields $1,148.70, so its estimate used an incorrect rate, taxable base, or both." +us,scenario_038,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model applied only CHIP's age and upper-income criteria. It omitted the prerequisite that a child must not qualify for Medicaid; this seven-year-old qualifies under Louisiana's OLDER_CHILD Medicaid category and is therefore ineligible for CHIP. +us,scenario_038,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated income below roughly 250% FPL as sufficient for CHIP. It failed to evaluate Medicaid first, where the child qualifies under the OLDER_CHILD category, which automatically bars CHIP eligibility." +us,scenario_038,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model used the CHIP upper-income limit as a standalone eligibility test, including a self-employment-tax adjustment that does not resolve the controlling issue. The child qualifies for Medicaid under Louisiana's OLDER_CHILD pathway, so CHIP eligibility is false regardless of being below CHIP's ceiling." +us,scenario_038,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model concluded that age, income below 250% FPL, and no listed insurance established CHIP eligibility. It omitted CHIP's Medicaid-exclusion rule: the child is Medicaid-eligible under Louisiana's OLDER_CHILD category and therefore cannot qualify for CHIP." +us,scenario_038,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated LaCHIP and Medicaid into a single affirmative eligibility result. The child's low income establishes Medicaid eligibility under the OLDER_CHILD category, and Medicaid eligibility specifically makes the child ineligible for CHIP." +us,scenario_038,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required submission contract." +us,scenario_038,child1_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model combined the CHIP and Medicaid thresholds and treated being below their income limits as CHIP eligibility. At this income, the seven-year-old qualifies for Louisiana Medicaid under the OLDER_CHILD category, and that prior eligibility excludes the child from CHIP." +us,scenario_038,child1_head_start_eligible,claude-opus-4.7,llm_error,age_disability,False,The model recognized that age seven exceeds Head Start’s typical preschool range of ages three through five but then incorrectly treated income eligibility as overriding the age requirement. A seven-year-old is not eligible for this preschool-age Head Start output. +us,scenario_038,child1_head_start_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model correctly stated that Head Start serves preschool-age children ages three through five, yet returned eligible for a seven-year-old based solely on household income. The unmet age condition independently requires a value of 0." +us,scenario_038,child1_head_start_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model explicitly and incorrectly classified age seven as within the Head Start preschool-age range. Head Start’s preschool pathway covers younger children, so Child 1 fails the age requirement regardless of the household’s income." +us,scenario_038,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply Louisiana's Medicaid older-child pathway to the 7-year-old. At household MAGI of 0.66 times FPL, the child satisfies the category's 2026 income limit, and the bank account and vehicle values are irrelevant to this MAGI test." +us,scenario_038,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age seven as within WIC’s eligible child age range. WIC covers children under age five, so Child 1 fails the categorical age test." +us,scenario_038,child2_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model applied CHIP's age and income conditions as sufficient and added an irrelevant asset analysis, but never checked the prerequisite that the child not qualify for Medicaid. Child 2 qualifies for Medicaid under the OLDER_CHILD category, which makes CHIP eligibility false." +us,scenario_038,child2_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,The model concluded that income below its stated CHIP limit directly establishes CHIP eligibility. It omitted the Medicaid-eligibility screen: Child 2 qualifies for Medicaid under the OLDER_CHILD category and is therefore excluded from CHIP. +us,scenario_038,child2_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,The model treated being age 6 and below the CHIP income ceiling as sufficient for CHIP coverage. It failed to apply the rule excluding children who qualify for Medicaid; Child 2 is Medicaid-eligible under the OLDER_CHILD category. +us,scenario_038,child2_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model checked age, lack of listed insurance, and a CHIP income ceiling but failed to check Medicaid eligibility. Child 2 qualifies for Medicaid under the OLDER_CHILD category, and that eligibility bars CHIP regardless of being uninsured or below the CHIP income ceiling." +us,scenario_038,child2_chip_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model used household income and the absence of an asset test to award LaCHIP without applying Medicaid precedence. Child 2 qualifies for Medicaid under the OLDER_CHILD category, so the child cannot qualify for CHIP." +us,scenario_038,child2_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,The model treated the under-19 and CHIP-income-limit tests as sufficient. It omitted the required Medicaid exclusion: Child 2 is Medicaid-eligible under the OLDER_CHILD category and therefore not CHIP-eligible. +us,scenario_038,child2_head_start_eligible,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model counted a five-person household even though the listed household has only the head, spouse, and two children. It then applied that inflated household-size poverty threshold and treated age six as preschool age, producing eligibility despite the failed Head Start rules." +us,scenario_038,child2_head_start_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly recognized that Head Start preschool ordinarily serves ages three through five, then invented an unlisted exception by assuming the six-year-old had not entered kindergarten. The prompt requires unlisted statuses to be false, so that exception cannot establish eligibility." +us,scenario_038,child2_head_start_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model acknowledged that the child had aged out at six but nevertheless assumed the child had not entered kindergarten. No delayed-entry status was provided, and the prompt forbids inferring it, so the model improperly overrode the preschool-age restriction." +us,scenario_038,child2_head_start_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly classified a six-year-old as a preschool-age Head Start child and treated its asserted poverty comparison as sufficient. Head Start requires the child-level age condition as well as the applicable income condition, and this child does not qualify under PolicyEngine’s traced rules." +us,scenario_038,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply Louisiana's older-child Medicaid pathway to the 6-year-old and incorrectly treated the household's income or assets as disqualifying. At MAGI of 0.66 times FPL, child2 passes the applicable older-child income threshold, and the $500 bank balance and vehicle value are not part of this MAGI eligibility test." +us,scenario_038,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly interpreted “birth to age 5” as including a six-year-old, even though WIC eligibility for children ends at the fifth birthday. It also used an income-limit estimate near $52,000 instead of the applicable PolicyEngine threshold, but Child 2 already fails the categorical age requirement." +us,scenario_038,child2_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model acknowledged that WIC covers children only through the fifth birthday but then treated age six as eligible. Household income cannot overcome Child 2’s failure of the categorical age requirement. +us,scenario_038,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model’s sole eligibility basis was the false assertion that age six falls within the benchmark’s WIC child age range. WIC child eligibility ends at the fifth birthday, so Child 2 is not eligible." +us,scenario_038,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly placed a six-year-old within the modeled WIC child age category. Because eligibility ends at the fifth birthday, Child 2 fails the categorical age test." +us,scenario_038,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable output for the requested variable and therefore failed the required submission contract. +us,scenario_038,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's reasoning correctly derived zero taxable income, zero tax before credits, and zero nonrefundable credits used, but its submitted value of -$1,644 contradicts that derivation. It inserted an unsupported negative amount even though nonrefundable credits cannot reduce liability below zero and refundable credits are excluded from this output." +us,scenario_038,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model replaced the 2026 two-child EITC maximum of $7,316 with speculative alternatives of $7,428 and $7,239, then submitted a number inconsistent with either calculation. The correct components are $7,316 of EITC and $2,909.35 of refundable CTC." +us,scenario_038,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly phased out the EITC even though this joint filer's income is within the two-child maximum-credit plateau. It also treated the refundable CTC as the full $3,400 cap instead of applying the earned-income phase-in, which yields $2,909.35." +us,scenario_038,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model arbitrarily reduced the refundable CTC from its own correctly computed $2,909 phase-in amount to $2,383 and understated the EITC. Applying the $7,316 EITC maximum and $2,909.35 refundable CTC yields the requested total." +us,scenario_038,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model reduced the plateau EITC to about $5,500 without applying a valid phaseout and cut the refundable CTC to $1,527 despite computing an earned-income limit near $3,074. The trace instead gives $7,316 of EITC and $2,909.35 of refundable CTC." +us,scenario_038,federal_refundable_credits,claude-opus-5,llm_error,other,False,"The model stated component estimates near $7,000 and $3,074 but submitted $9,199, which does not equal those components. It also failed to use the exact $7,316 EITC and the $2,909.35 refundable CTC phase-in." +us,scenario_038,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the 2025 EITC maximum of $6,604 as a proxy instead of the applicable 2026 maximum of $7,316. Its refundable CTC calculation was essentially the correct $2,909 amount, so the stale EITC parameter accounts for the shortfall." +us,scenario_038,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's stated estimates of roughly $6,900 EITC and $2,896 refundable CTC imply about $9,796, yet it submitted $8,500 after an unsupported adjustment. The correct computation uses $7,316 and $2,909.35 with no such reduction." +us,scenario_038,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied obsolete refundable CTC rules: a $1,000-per-child cap and a $3,000 earnings threshold. The applicable earned-income phase-in produces $2,909.35 of refundable CTC, which combines with the $7,316 EITC." +us,scenario_038,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model limited refundable CTC to $1,000 per child and used a $3,000 phase-in threshold. The applicable refundable CTC is $2,909.35, not $2,000, and the exact EITC is $7,316." +us,scenario_038,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model treated the full $4,000 nominal CTC as refundable and understated the plateau EITC as $4,100. Only $2,909.35 of CTC is refundable under the earned-income phase-in, while the EITC equals $7,316." +us,scenario_038,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $2,000 refundable CTC without applying the 15% earned-income phase-in, which produces $2,909.35. It also rounded the applicable $7,316 EITC down to $7,300." +us,scenario_038,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied an obsolete $1,000-per-child refundable CTC limit, producing $2,000 instead of $2,909.35. It also approximated the two-child EITC as $7,250 rather than using $7,316." +us,scenario_038,federal_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model named the correct two credit categories but supplied no component calculation supporting $9,753.44. The engine computation is $7,316 of EITC plus $2,909.35 of refundable CTC." +us,scenario_038,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. +us,scenario_038,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model supplied only an unsupported combined estimate and did not calculate either credit under the applicable rules. The required component calculation is $7,316 of EITC plus $2,909.35 of refundable CTC." +us,scenario_038,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly denied refundable-credit eligibility despite two qualifying children and substantial earned income. Those facts generate a $7,316 EITC and $2,909.35 refundable CTC even with zero regular income-tax liability." +us,scenario_038,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model correctly identified the $7,316 EITC but treated the refundable CTC as a flat $1,800 per child. The refundable portion is limited by the earned-income phase-in to $2,909.35, not the aggregate per-child ceiling of $3,600." +us,scenario_038,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used $8,231 as the two-child EITC maximum instead of $7,316. It also computed refundable CTC from gross wages plus self-employment income, producing $3,073.95 rather than the traced $2,909.35." +us,scenario_038,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model correctly used the $7,316 EITC but understated the refundable CTC as $2,731.32. Applying the refundable CTC earned-income phase-in yields $2,909.35." +us,scenario_038,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model confused zero regular income-tax liability with zero refundable credits and disregarded the listed earned income and qualifying children. Refundability produces $7,316 of EITC and $2,909.35 of CTC independently of positive pre-credit income tax." +us,scenario_038,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model imposed a $2,000 aggregate refundable CTC ceiling instead of applying the applicable per-child limit and earned-income phase-in. That phase-in yields $2,909.35, and the EITC is $7,316 rather than $7,312." +us,scenario_038,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used post-TCJA-reversion assumptions of a $1,000-per-child refundable amount and a $3,000 earnings threshold. The applicable 2026 rules yield $2,909.35 of refundable CTC, while the exact two-child EITC maximum is $7,316 rather than $7,338." +us,scenario_038,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model overstated the two-child EITC as about $7,422 instead of $7,316 and understated refundable CTC as about $2,879 instead of $2,909.35. Both component approximations used the wrong 2026 parameter computation." +us,scenario_038,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. +us,scenario_038,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly used the $7,316 EITC but calculated refundable CTC from the unadjusted $22,993 sum of wages and self-employment income. The applicable earned-income computation produces $2,909.35 rather than $3,073.95." +us,scenario_038,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model understated the EITC as roughly $6,500 and treated the refundable CTC as the full $3,400 per-child-cap total. The household receives the $7,316 EITC maximum, while the CTC earned-income phase-in limits refundability to $2,909.35." +us,scenario_038,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model repeatedly mixed three-child and two-child EITC maxima, used a phaseout threshold that does not apply to this joint filer, and ultimately discarded its own component calculation to submit $6,868.75. The household is on the two-child joint-filer plateau, producing $7,316 of EITC plus $2,909.35 of refundable CTC." +us,scenario_038,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly placed $22,993 of income near the end of the married-joint two-child EITC phaseout and reduced the EITC to $46. The household remains on the maximum-credit plateau and receives $7,316 of EITC; its refundable CTC is $2,909.35 rather than the rounded $3,000." +us,scenario_038,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model asserted nonqualification without applying either qualifying pathway. PolicyEngine derives categorical eligibility from the household's $7,286.94 annual SNAP benefit and independently derives income eligibility because the school-meal FPG ratio is 0.70, below 1.30." +us,scenario_038,free_school_meals_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model correctly computed gross income below 130% FPG but then incorrectly imposed SNAP receipt or direct certification as an additional requirement and treated an unlisted SNAP input as proof of no SNAP participation. Income at or below 130% FPG independently qualifies the children for free meals, and PolicyEngine also computes $7,286.94 of SNAP under the assumed-take-up instruction, which supplies categorical eligibility and produces positive meal support." us,scenario_038,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_038,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,health_coverage,False,"The model applied the wrong Louisiana adult Medicaid income threshold and concluded the household income exceeded it. PolicyEngine classifies the 25-year-old head as an ACA expansion adult and uses MAGI at 0.66 times FPL, which is below the expansion adult limit." -us,scenario_038,head_medicaid_eligible,gpt-5.4-nano,llm_error,health_coverage,False,"The model treated the provided income and assets as disqualifying for Medicaid instead of applying Louisiana's ACA expansion adult MAGI pathway. For a 25-year-old non-dependent head, PolicyEngine uses the ADULT category and counts MAGI at 0.66 times FPL, with the listed bank assets and vehicle value not disqualifying under that MAGI eligibility test." -us,scenario_038,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a parent/guardian WIC eligibility pathway that does not apply to the head. It treated household income below 185% FPL and the presence of children as enough, but WIC requires the applicant to be pregnant, postpartum, breastfeeding, an infant, or a child under age 5, and the head is none of those." -us,scenario_038,head_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model inferred unlisted pregnancy, postpartum, or breastfeeding status and also misstated the household as having children under 5. Under the prompt, unlisted statuses are false and the children are ages 7 and 6, so neither categorical WIC eligibility nor adjunctive eligibility creates head eligibility." -us,scenario_038,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated being a 25-year-old adult as sufficient for the pregnant/postpartum-age WIC category. WIC requires actual pregnancy, postpartum status, or breastfeeding status for an adult applicant, and those facts are unlisted and therefore false." -us,scenario_038,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,The model did not submit a payroll_tax value or supporting explanation. This is a missing-output failure rather than a substantive payroll-tax computation. -us,scenario_038,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model described the correct employee-side wage-only FICA base but submitted $1,140, which doubles the employee-side amount and is consistent with adding an employer-side FICA layer that the prompt explicitly excludes. It also failed to reconcile its stated Social Security and Medicare calculation on $7,468 of wages with the value it submitted." -us,scenario_038,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified the two employee-side FICA components but then discarded them and submitted $246.08 under an invented benchmark exclusion. Employee Social Security and employee Medicare taxes are both included in payroll_tax, so the wage-only FICA components total $571.26 rather than $246.08." -us,scenario_038,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not submit a payroll_tax value or supporting explanation. This is a missing-output failure rather than a substantive payroll-tax computation. -us,scenario_038,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model applied the right wage-only employee FICA structure but miscomputed and misreported the cents: Social Security is $462.98 in the engine trace and Medicare is $108.28, totaling $571.26. Its submitted $569.90 also contradicts its own stated total of $571.30, creating an arithmetic and final-answer mismatch." -us,scenario_038,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model treated 185% of the federal poverty guideline as a standalone maximum and ignored the lower boundary created by the free-meals tier. The household is at 70% of the guideline and categorically eligible, so it qualifies for free meals rather than reduced-price meals." -us,scenario_038,reduced_price_school_meals_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model used incorrect poverty-guideline thresholds to place $22,993 between 130% and 185%, even though the engine ratio is 70%. It also inferred no categorical eligibility from the prompt's lack of listed SNAP or TANF receipt, while the applicable computation establishes categorical eligibility; either free-tier pathway excludes reduced-price support." -us,scenario_038,self_employment_tax,claude-fable-5,llm_error,other,False,"The model correctly described the self-employment tax formula and even derived $2,193.61, but it submitted $2,196.28 instead. Its error is a final-answer transcription failure: the numeric value field does not match its own stated computation." -us,scenario_038,self_employment_tax,claude-haiku-4.5,llm_error,other,False,"The model applied the right 92.35% self-employment tax base rule and 15.3% rate, but miscomputed $15,525 × 0.9235 as $14,339.59 instead of $14,337.34. That inflated the taxable self-employment earnings and produced $2,194.96 rather than $2,193.61." -us,scenario_038,self_employment_tax,gpt-5.4-mini,llm_error,other,False,"The model used an unsupported rounded shortcut for the standard self-employment tax computation and submitted $2,197.00 without carrying the 92.35% base and 15.3% rate to cents. The correct calculation on $14,337.3375 of net self-employment earnings yields $2,193.61." -us,scenario_038,self_employment_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model stated the correct 92.35% base and 15.3% combined rate, but its $1,360.68 answer is not the result of applying that rate to $14,337.34. It effectively used a much lower tax rate, omitting a substantial part of the Social Security and Medicare self-employment tax liability." -us,scenario_038,self_employment_tax,grok-4.3,llm_error,other,False,"The model treated the calculation as an approximation and rounded the self-employment tax down to $2,192.00 instead of carrying the statutory 92.35% base reduction and 15.3% rate through the cent-level computation. The exact formula gives $2,193.61." -us,scenario_038,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model provided no parseable self_employment_tax answer or supporting explanation. This is a missing-output failure rather than a substantive tax-rule computation. -us,scenario_038,snap,claude-fable-5,llm_error,other,False,"The model's own calculation produced about $7,236 annually, but it submitted $4,200 with no supporting computation. It violated the answer contract by replacing its calculated result with an unrelated number." -us,scenario_038,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented a $400 monthly shelter deduction from the mortgage balance despite the instruction that unlisted mortgage payments, taxes, insurance, and utilities are zero. It then abandoned its own $7,772.16 calculation and submitted an unsupported $1,008." -us,scenario_038,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model inconsistently computed a roughly $540 monthly benefit and then asserted roughly $759 per month without deriving that change. The correct maximum allotment is $994 and the expected contribution is $392.70, producing $601.30 monthly rather than the submitted $759.50." -us,scenario_038,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model estimated the deductions instead of applying the traced net income of $1,309.83. With the $994 maximum and $392.70 expected contribution, the monthly benefit is $601.30, not $654." -us,scenario_038,snap,claude-opus-5,llm_error,thresholds_rates,False,The model used an unsupported $564 monthly estimate rather than calculating the allotment from the applicable parameters. The $994 maximum less the $392.70 contribution yields $601.30 per month. -us,scenario_038,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model deliberately used the 2025 maximum allotment of $975 instead of the applicable $994 amount and used a $204 standard deduction rather than the deduction embedded in the $1,309.83 traced net income. Those stale parameters reduced the result to $576 monthly instead of $601.30." -us,scenario_038,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced self-employment income by 20% before separately applying the SNAP earned-income deduction and also invoked unlisted dependent-care and medical deductions. The traced net income is $1,309.83, so the benefit is $601.30 monthly." -us,scenario_038,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an unsupported $1,050 maximum allotment instead of $994. Applying 30% to the traced $1,309.83 net income produces a $392.70 contribution and a $601.30 monthly benefit." -us,scenario_038,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented a $310 monthly dependent deduction even though no dependent-care expense was listed. It also used a 2024 maximum allotment rather than the applicable $994, so its net income and benefit were both miscomputed." -us,scenario_038,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $9,000 implies a $750 monthly allotment, but the model supplied no calculation supporting that amount. The applicable formula yields $994 - 30% × $1,309.83 = $601.30 monthly." -us,scenario_038,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model named the correct general formula but did not apply the applicable $994 maximum and $1,309.83 net income. Those inputs yield $7,286.94 annually, not $6,878." -us,scenario_038,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model approximated the maximum allotment as $990 and net income as $1,333. The applicable values are $994 and $1,309.83, yielding $601.30 monthly rather than $590." -us,scenario_038,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted $6,276 implies a $523 monthly benefit without applying the stated maximum-allotment formula. The traced calculation produces $601.30 monthly and $7,286.94 annually." -us,scenario_038,snap,glm-5.2,llm_error,thresholds_rates,False,"The model estimated the maximum allotment as $1,020 rather than using $994 and estimated the standard deduction rather than using the traced $1,309.83 net income. These parameter errors inflated the monthly benefit to $626.14 instead of $601.30." -us,scenario_038,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly concluded that earnings or assets eliminate the benefit. The household has only $500 in assets, passes the gross and net income tests, and is categorically eligible through TANF non-cash assistance, producing a positive $7,286.94 annual benefit." -us,scenario_038,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly treated the household as ineligible and returned zero. The household passes the income tests and has categorical eligibility through TANF non-cash assistance, after which the allotment formula yields $601.30 monthly." -us,scenario_038,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model used an $879 maximum allotment instead of the applicable $994. With the traced contribution of $392.70, the benefit is $601.30 monthly rather than about $484." -us,scenario_038,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The answer is consistent with a slightly incorrect maximum allotment, deduction, or rounding step. The exact traced inputs produce ($994 - $392.70) × 12 = $7,286.94, not $7,316.88." -us,scenario_038,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model rounded the monthly benefit to $601 before annualizing it. PolicyEngine retains the unrounded monthly result of about $601.30, which annualizes to $7,286.94 rather than $7,212." -us,scenario_038,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model annualized a prematurely rounded $601 monthly allotment. The $994 maximum less the $392.70 contribution is about $601.30 per month, yielding $7,286.94 annually." -us,scenario_038,snap,grok-4.3,llm_error,missing_output,False,The model treated the no-inference instruction as a reason to replace a calculable program output with zero. The benchmark supplies the household facts while the applicable SNAP parameters determine a $601.30 monthly allotment. -us,scenario_038,snap,grok-4.5,llm_error,thresholds_rates,False,"The model projected the maximum allotment as roughly $1,000 instead of using $994 and relied on rounded deduction estimates. The exact expected contribution is $392.70, leaving $601.30 monthly and $7,286.94 annually." -us,scenario_038,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an annual standard deduction of roughly $3,985 and an annual maximum allotment of roughly $12,337, neither of which matches the applicable SNAP parameters. The traced deductions produce $1,309.83 monthly net income, and the applicable annual maximum is $11,928." -us,scenario_038,snap,kimi-k2.6,llm_error,thresholds_rates,False,"The model used a $1,020 maximum allotment rather than $994 and estimated net income as $1,325 rather than $1,309.83. Those errors raised the monthly benefit to $622 instead of $601.30." -us,scenario_038,snap,kimi-k3,llm_error,thresholds_rates,False,"The model got net income essentially right but used a $997 maximum allotment rather than $994. Subtracting the $392.70 expected contribution from $994 yields about $601.30 monthly, not $604.04." -us,scenario_038,snap,minimax-m3,llm_error,thresholds_rates,False,"The model used an unsupported $1,050 maximum allotment and overstated net income as about $1,329. The applicable $994 maximum and $1,309.83 net income yield $601.30 monthly rather than about $651." -us,scenario_038,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model first calculated about $572.54 monthly, then inserted unspecified shelter and other adjustments and replaced that result with $879 monthly. No shelter expense was listed, and the traced deductions yield $1,309.83 net income and a $601.30 monthly allotment." -us,scenario_038,spouse_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model treated age under 19 and income below the Louisiana CHIP ceiling as sufficient. It omitted the Medicaid-exclusion step: the spouse is Medicaid-eligible under the OLDER_CHILD category and therefore ineligible for CHIP. -us,scenario_038,spouse_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model interpreted the absence of listed Medicaid coverage as the absence of Medicaid eligibility, despite the requested output concerning eligibility rather than enrollment. The spouse qualifies for Medicaid under the OLDER_CHILD category, which automatically bars CHIP eligibility." -us,scenario_038,spouse_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model stopped after finding that the spouse was under 19 and below the LaCHIP income limit. It failed to test Medicaid first; the spouse is Medicaid-eligible under the OLDER_CHILD category, so CHIP is unavailable regardless of the CHIP income ceiling." -us,scenario_038,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no spouse_chip_eligible value or explanation, violating the required structured-output contract." -us,scenario_038,spouse_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model classified the 18-year-old spouse as a general adult or parent and applied the Louisiana adult Medicaid income threshold. It missed PolicyEngine's OLDER_CHILD Medicaid category for an age-18 person, under which the spouse's 0.66 FPL MAGI is within the applicable limit." -us,scenario_038,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the spouse's Medicaid eligibility as failing under a generic income/assets screen instead of applying the age-based OLDER_CHILD Medicaid pathway. The correct computation uses the spouse's age 18 category and 0.66 FPL MAGI, which satisfies Louisiana Medicaid eligibility under PolicyEngine rules." -us,scenario_038,spouse_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a caregiver-of-dependent-children WIC category and also used the wrong household size by applying a household-of-5 threshold to a four-person household. The spouse is not pregnant, postpartum, or breastfeeding, and the children are 6 and 7 rather than under 5, so income below 185% FPL does not make the spouse WIC-eligible." -us,scenario_038,spouse_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly named the adult WIC categories but then replaced them with an unsupported assumption that a woman of childbearing age in a low-income household with children qualifies. The facts do not list pregnancy, postpartum status, or breastfeeding, so the spouse fails categorical eligibility regardless of household income." -us,scenario_038,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model acknowledged that the spouse lacks pregnancy, postpartum, or breastfeeding status, then incorrectly treated income eligibility and being a woman of childbearing age in a family as a substitute categorical pathway. Adjunctive or income eligibility cannot override the missing WIC categorical status for the spouse." -us,scenario_038,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model inferred pregnancy, postpartum, or breastfeeding status from the spouse's age and household composition even though the prompt says unlisted facts are false. With no listed qualifying adult WIC status and no child under age 5 in the household, the spouse is categorically ineligible despite low income." -us,scenario_038,spouse_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated being age 18 and living with young children as the benchmark's WIC eligibility rule. PolicyEngine requires an actual WIC categorical status for the person, and the spouse has no pregnancy, postpartum, or breastfeeding fact while the children are ages 6 and 7, so the spouse is not eligible." -us,scenario_038,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own calculation reduced Louisiana taxable income to zero and explicitly concluded that the tax was $0, but it submitted $218. This is an internal answer-selection failure: the numeric output contradicts its completed derivation." -us,scenario_038,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $9,000 Louisiana standard deduction and consequently treated $13,696 as taxable at 1.85%. It failed to apply the 2026 joint-filer and dependent provisions that eliminate Louisiana taxable income for this household." -us,scenario_038,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model deducted only $9,000 for the joint return and $2,000 for two dependents, leaving a fictitious $10,696 tax base. Louisiana's applicable 2026 joint-filer and dependent provisions reduce the household's taxable income to zero." -us,scenario_038,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,The submitted $152 implies that the model left positive Louisiana taxable income after its vaguely stated personal exemptions and lower-bracket calculation. Correct application of the 2026 joint-filer and two-dependent deductions leaves no Louisiana taxable income and therefore no pre-refundable-credit tax. -us,scenario_038,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used a $9,000 standard deduction plus only $4,000 of personal exemptions, producing $8,896 of taxable income. Those deductions and exemptions do not reflect the applicable 2026 treatment for this married couple with two dependents, which reduces Louisiana taxable income to zero." -us,scenario_038,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied the 3% rate to an implied positive tax base of about $9,400 after an understated married deduction. The applicable 2026 joint-filer and dependent provisions eliminate the tax base before the rate is applied." -us,scenario_038,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied a purported 1.75% lowest bracket even though its own description identifies the post-reform regime; Louisiana's 2026 individual income tax uses a flat 3% structure rather than that bracket. It also constructed $3,896 of taxable income from an incorrect exemption-only shortcut instead of applying the provisions that reduce taxable income to zero." -us,scenario_038,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required structured-output contract rather than producing a substantive Louisiana tax estimate. -us,scenario_038,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used obsolete 2% and 4% Louisiana brackets and an $11,000 exemption scheme to create $10,896 of taxable income. For 2026, that bracket calculation is inapplicable, and the applicable joint-filer and dependent provisions reduce taxable income to zero." -us,scenario_038,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model treated the household as remaining at the estimated 2026 maximum federal EITC and used $7,478 instead of calculating the applicable $7,316 credit. Applying Louisiana's 5% rate to the correct federal EITC yields $365.80, not $374." -us,scenario_038,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Louisiana has no refundable EITC. This household receives a refundable Louisiana EITC equal to 5% of its $7,316 federal EITC, producing $365.80." -us,scenario_038,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model understated the federal EITC as approximately $3,680 even though the applicable joint-filer credit with two children is $7,316. Its erroneous federal credit halved the Louisiana 5% EITC from $365.80 to about $184." -us,scenario_038,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model contradicted its own near-plateau reasoning by substituting a federal EITC of only $2,640. The applicable federal EITC is $7,316, and Louisiana's 5% refundable credit is therefore $365.80." -us,scenario_038,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model rounded the federal EITC to an unsupported $7,000 when estimating the 5% Louisiana credit. The exact federal EITC is $7,316, so the Louisiana EITC is $365.80 rather than $350." -us,scenario_038,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly claimed that Louisiana eliminated its refundable EITC. Louisiana's applicable refundable EITC equals 5% of the household's $7,316 federal EITC, or $365.80." -us,scenario_038,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly classified Louisiana's EITC as nonrefundable. The credit is refundable and equals 5% of the $7,316 federal EITC, producing $365.80." -us,scenario_038,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used an incorrect federal EITC of $7,354. The applicable federal EITC is $7,316, and multiplying it by Louisiana's 5% rate gives $365.80 rather than $367.70." -us,scenario_038,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted Louisiana's refundable EITC despite the household qualifying for it. The credit is 5% of the $7,316 federal EITC, yielding $365.80." -us,scenario_038,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model correctly identified Louisiana's 5% rate but understated the federal EITC as $7,250. The federal credit is $7,316, making the state refundable credit $365.80 rather than $362.50." -us,scenario_038,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The submitted $353.43 implies a federal EITC of $7,068.60 at Louisiana's 5% rate. The applicable federal EITC is $7,316, so the state refundable EITC is $365.80." +us,scenario_038,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model applied an incorrect Louisiana Medicaid income threshold to the head. As a 25-year-old ACA expansion adult with MAGI at 0.66 FPL, the head is below the applicable expansion threshold and is eligible." +us,scenario_038,head_medicaid_eligible,gpt-5.4-nano,llm_error,asset_resource,False,"The model improperly treated the provided assets as relevant to this eligibility determination and failed to apply Louisiana's MAGI-based ACA adult expansion pathway. That pathway has no asset test, and the head's MAGI income level of 0.66 FPL qualifies." +us,scenario_038,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a WIC categorical pathway for a parent or guardian of children. Only pregnant, postpartum, or breastfeeding women, infants, and children under age 5 qualify categorically; the head has no qualifying status and both children are over age 5." +us,scenario_038,head_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated parent or caretaker status as WIC categorical eligibility and also misstated the ages 7 and 6 children as under 5. Low income and adjunctive eligibility through SNAP or Medicaid satisfy an income pathway only; they do not replace the required pregnant, postpartum, breastfeeding, infant, or under-5 category." +us,scenario_038,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model converted reproductive age into pregnant or postpartum status despite the prompt directing that unlisted statuses are false. A 25-year-old adult is not WIC-eligible without an actual pregnant, postpartum, or breastfeeding status, and assumed program take-up does not establish categorical eligibility." +us,scenario_038,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output, violating the required structured-output contract." +us,scenario_038,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model stated that only employee-side FICA applied but submitted $1,140, approximately twice the employee liability and therefore consistent with improperly adding an employer-side FICA amount. The requested output excludes employer payroll taxes and equals $571.26." +us,scenario_038,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified Social Security and Medicare as the applicable employee components, then arbitrarily reduced their stated sum instead of adding them. The trace yields $462.98 plus $108.28, so no benchmark exclusion produces its $246.08 result." +us,scenario_038,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output, violating the required structured-output contract." +us,scenario_038,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model’s submitted $569.90 contradicts its own stated component sum of $571.30, so it failed to carry its calculation into the output. It also recomputed Social Security from the displayed rounded wage figure instead of using the engine’s component result of $462.98; adding that to $108.28 gives $571.26." +us,scenario_038,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model treated 185% of the federal poverty guideline as a one-sided reduced-price eligibility ceiling and failed to apply the free-tier test first. The household is at 70% of the guideline and categorically eligible, so it qualifies for free meals and cannot receive reduced-price support." +us,scenario_038,reduced_price_school_meals_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model used incorrect 130% and 185% poverty thresholds, causing it to place $22,993 inside the reduced-price band even though the engine's ratio is 70%. It also inferred no categorical eligibility from the prompt despite assumed program take-up; the household meets categorical eligibility and receives the mutually exclusive free-meals tier." +us,scenario_038,self_employment_tax,claude-fable-5,llm_error,other,False,"The model’s stated computation produces $2,193.61, but it submitted $2,196.28 instead. It failed to carry its correctly calculated result into the numeric output." +us,scenario_038,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model multiplied $15,525 by 92.35% incorrectly: the product is $14,337.34, not $14,339.59. Applying 15.3% to the correct base yields $2,193.61." +us,scenario_038,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The submitted $2,197 does not result from the standard computation it invoked. Applying 92.35% and then 15.3% to $15,525 yields $2,193.61, so the model introduced an unsupported approximation or arithmetic error." +us,scenario_038,self_employment_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model claimed to apply 15.3% to 92.35% of $15,525, but that calculation yields $2,193.61 rather than $1,360.68. Its answer therefore failed to apply the stated combined Social Security and Medicare rate to the full taxable self-employment earnings base." +us,scenario_038,self_employment_tax,grok-4.3,llm_error,other,False,"The model rounded an exact annual tax calculation down to $2,192 without supporting arithmetic. The 92.35% base is $14,337.34, and applying 15.3% produces $2,193.61." +us,scenario_038,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self-employment-tax output or explanation, so the required numeric value was missing." +us,scenario_038,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model incorrectly fed the deduction for one-half of self-employment tax back into the self-employment-tax base. That deduction reduces adjusted gross income for income-tax purposes only; self-employment tax remains 15.3% of $14,337.34, or $2,193.61." +us,scenario_038,snap,claude-fable-5,llm_error,other,False,"The model's own calculation produced about $7,236, but it submitted $4,200 with no computation supporting that figure. It discarded its derived monthly benefit instead of annualizing the correct $601.30 monthly amount to $7,286.94." +us,scenario_038,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented a $400 monthly shelter deduction from a mortgage balance despite the prompt setting unlisted mortgage payments, taxes, insurance, and utilities to zero. It then abandoned its own $7,772.16 calculation and asserted an unsupported $84 monthly benefit." +us,scenario_038,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used estimated standard-deduction and allotment parameters, then changed a calculated benefit of about $540 per month to about $759 per month without an intervening rule or deduction. The correct parameters yield $1,309.83 net income and $601.30 monthly SNAP." +us,scenario_038,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model estimated the standard deduction and then supplied an approximate annual benefit without completing the contribution calculation. Applying the exact $994 allotment and $392.70 contribution yields $7,286.94, not $7,848." +us,scenario_038,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model replaced the exact SNAP computation with an unsupported midpoint estimate of $564 per month. The specified deductions produce $1,309.83 net monthly income and a $601.30 monthly benefit." +us,scenario_038,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used FY2025's $975 maximum allotment and projected deduction values instead of the applicable $994 allotment and exact deductions. It also rounded the monthly result before annualizing, whereas the trace annualizes $601.30 to $7,286.94." +us,scenario_038,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced self-employment income by 20% before constructing gross income and then invoked dependent-care and medical deductions that the facts do not support. SNAP applies the earned-income deduction in the net-income calculation, producing $1,309.83 net monthly income." +us,scenario_038,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an invented $1,050 maximum allotment instead of the applicable $994. With the exact $392.70 contribution, the monthly benefit is $601.30 rather than $652.54." +us,scenario_038,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented a $310 monthly dependent deduction even though no dependent-care expense was listed. It also used a 2024 allotment rather than the applicable $994, so its $1,025 net-income figure and resulting benefit are wrong." +us,scenario_038,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model gave only a broad eligibility estimate and did not apply the maximum-allotment-minus-30%-of-net-income formula. The exact computation is $994 minus $392.70 per month, annualized to $7,286.94." +us,scenario_038,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model named the general formula but did not use the exact $1,309.83 net income, $994 allotment, or $392.70 contribution. Those inputs produce $601.30 monthly and $7,286.94 annually." +us,scenario_038,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model approximated net income as $1,333 and the maximum allotment as $990. The exact values are $1,309.83 and $994, yielding $601.30 per month rather than $590." +us,scenario_038,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model established eligibility but omitted the numerical benefit derivation and submitted an unsupported amount. The benefit calculation yields $601.30 monthly and $7,286.94 annually." +us,scenario_038,snap,glm-5.2,llm_error,thresholds_rates,False,"The model substituted estimated values of $220 for the standard deduction and $1,020 for the maximum allotment. The applicable parameters produce $1,309.83 net income and a $601.30 monthly allotment." +us,scenario_038,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model incorrectly denied a positive benefit despite the household passing the income and asset requirements and qualifying categorically through TANF non-cash assistance. Its eligible monthly allotment is $601.30. +us,scenario_038,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model incorrectly treated the household as receiving no SNAP. The household is categorically eligible through TANF non-cash assistance and its income calculation produces a positive $601.30 monthly allotment. +us,scenario_038,snap,gpt-5.5,llm_error,thresholds_rates,False,The model used an $879 four-person maximum allotment instead of $994. Using the correct allotment and $392.70 expected contribution produces $601.30 monthly. +us,scenario_038,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model invoked the correct structure but used imprecise deduction or allotment inputs. The exact $994 maximum less the $392.70 contribution is $601.30 monthly, totaling $7,286.94." +us,scenario_038,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model rounded the monthly benefit to $601 before multiplying by 12. The trace retains the unrounded $601.30 monthly amount, producing $7,286.94 rather than $7,212." +us,scenario_038,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model prematurely rounded the monthly allotment from $601.30 to $601. Annualizing the trace's unrounded monthly amount yields $7,286.94." +us,scenario_038,snap,grok-4.3,llm_error,other,False,"The model misread the instruction that unlisted household inputs equal zero as requiring an unknown program parameter to be treated as zero. SNAP parameters remain applicable, and the stated facts yield $601.30 monthly." +us,scenario_038,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used projected round figures of about $225 for the deduction and $1,000 for the maximum allotment. The exact calculation uses $1,309.83 net income and a $994 maximum, yielding $7,286.94 annually." +us,scenario_038,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model used approximate annual parameters, including an overstated maximum allotment of about $12,337 and an overstated standard deduction of about $3,985. The applicable maximum is $11,928 annually, and the exact monthly calculation yields $7,286.94." +us,scenario_038,snap,inkling,llm_error,thresholds_rates,False,"The model approximated net income as $1,316 and the maximum allotment as $1,000 rather than using $1,309.83 and $994. Those exact inputs yield $601.30 monthly, not $605." +us,scenario_038,snap,kimi-k2.6,llm_error,thresholds_rates,False,"The model used an estimated $1,020 maximum allotment instead of $994 and rounded the resulting monthly amount down before annualizing. The correct contribution is $392.70, leaving $601.30 per month." +us,scenario_038,snap,kimi-k3,llm_error,thresholds_rates,False,"The model closely matched net income but used a $997 maximum allotment instead of $994. The correct maximum less 30% of $1,309.83 produces $601.30 monthly and $7,286.94 annually." +us,scenario_038,snap,minimax-m3,llm_error,thresholds_rates,False,"The model used an estimated $1,050 maximum allotment instead of $994 and overstated net income. The applicable inputs yield a $601.30 monthly benefit rather than about $651." +us,scenario_038,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model first calculated about $572.54 monthly, then raised it to $879 by invoking unspecified shelter and other adjustments despite the prompt setting unlisted expenses to zero. No such adjustment supports its submitted $10,548; the exact monthly benefit is $601.30." +us,scenario_038,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced net income to zero through a shelter deduction even though no shelter payment or utility expense was listed. Net monthly income is $1,309.83, so the household receives $601.30 rather than the maximum allotment." +us,scenario_038,spouse_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model applied only the CHIP age and income conditions and skipped the prerequisite that the person not qualify for Medicaid. At age 18, the spouse is Medicaid-eligible under the OLDER_CHILD category, which makes the spouse ineligible for CHIP." +us,scenario_038,spouse_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model treated the absence of listed Medicaid coverage as the absence of Medicaid eligibility, despite the question asking about eligibility rather than enrollment. The spouse qualifies for Medicaid under the OLDER_CHILD category, and that eligibility bars CHIP even though the spouse is under 19 and below the CHIP income limit." +us,scenario_038,spouse_chip_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model concluded that satisfying the CHIP age and income ceiling was sufficient and never evaluated Medicaid eligibility first. The spouse is Medicaid-eligible under the OLDER_CHILD category, so the mutually exclusive CHIP pathway yields No." +us,scenario_038,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no spouse_chip_eligible output or explanation, violating the required output contract." +us,scenario_038,spouse_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated the 18-year-old spouse as an adult or parent and applied that category's income threshold. PolicyEngine instead places the spouse in the OLDER_CHILD category, under which MAGI of 0.66 times FPL establishes Medicaid eligibility." +us,scenario_038,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply the OLDER_CHILD eligibility pathway for an 18-year-old spouse and invoked income and assets generically. The spouse's 0.66-times-FPL MAGI is below the applicable older-child threshold, while assets are not part of this MAGI-category eligibility test." +us,scenario_038,spouse_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a WIC caregiver-of-dependent-children category, but WIC does not cover parents or caregivers merely because they live with children. It also counted a five-person household despite only four listed people; regardless of the income threshold, the spouse lacks pregnancy, breastfeeding, or postpartum status and is not categorically eligible." +us,scenario_038,spouse_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model replaced WIC's pregnancy, breastfeeding, or postpartum requirement with being a woman of childbearing age in a low-income household. Childbearing age and household income do not establish categorical eligibility, and no qualifying status was listed for the spouse." +us,scenario_038,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly recognized that the spouse lacked a qualifying categorical status, then contradicted that conclusion by treating low income and membership in a childbearing household as sufficient. Income or adjunctive income eligibility satisfies only the financial prong and does not create pregnancy, breastfeeding, postpartum, infant, or under-five status." +us,scenario_038,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model inferred that the spouse was pregnant, postpartum, or breastfeeding from her age and the presence of children, contrary to the instruction that unlisted statuses are false. The listed children are ages six and seven, and neither their presence nor low household income gives the spouse WIC categorical eligibility." +us,scenario_038,spouse_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The answer treats being 18 and living with children as a WIC eligibility pathway. The spouse has no listed pregnancy, breastfeeding, or postpartum status, and the children are six and seven rather than under five, so the required categorical basis is absent." +us,scenario_038,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model explicitly derived zero Louisiana taxable income and zero tax after its $25,000 joint standard deduction, then submitted $218 anyway. Its numeric output contradicts its own completed computation." +us,scenario_038,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $9,000 joint standard deduction and thereby manufactured $13,696 of taxable income. The applicable Louisiana calculation for this joint filer with two dependents leaves no taxable income, so applying 1.85% to that artificial base produced the erroneous $253.38." +us,scenario_038,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model combined a $9,000 joint deduction with only $2,000 of dependent exemptions, leaving an artificial $10,696 tax base. Louisiana’s applicable joint-filer and dependent treatment reduces taxable income to zero before any rate is applied." +us,scenario_038,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $152 estimate implies that the model retained a positive Louisiana taxable-income base after exemptions. The joint-filer and two-dependent calculation instead leaves zero taxable income, so no lower-bracket tax arises." +us,scenario_038,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used a $9,000 standard deduction plus $4,000 of personal exemptions and incorrectly retained $8,896 of taxable income. The applicable Louisiana deductions and dependent treatment eliminate the taxable base for this household, making its 1.85% calculation inapplicable." +us,scenario_038,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied Louisiana’s 3% rate to an implied positive base of about $9,400 after an unspecified married deduction. The applicable joint-filer and two-dependent calculation leaves zero taxable income, so the 3% rate has no positive base to tax." +us,scenario_038,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model substituted a $4,500-per-person exemption scheme and a 1.75% lowest bracket, producing $3,896 of taxable income. Those parameters do not govern this 2026 Louisiana calculation; the applicable joint-filer treatment leaves the household below the taxable-income threshold." +us,scenario_038,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required output contract rather than completing the Louisiana tax calculation. +us,scenario_038,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used an obsolete graduated 2% and 4% rate schedule together with an exemption-only taxable-income calculation. The applicable 2026 Louisiana joint-filer calculation leaves zero taxable income, so neither the $10,896 base nor the graduated-bracket tax applies." +us,scenario_038,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model treated the household as receiving an estimated maximum federal EITC of $7,478 instead of the computed $7,316. Applying Louisiana's 5% rate to the correct federal credit yields $365.80, not $374." +us,scenario_038,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly denied the existence of Louisiana's refundable EITC. This household receives 5% of its $7,316 federal EITC, producing a $365.80 refundable Louisiana credit." +us,scenario_038,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model understated the federal EITC as approximately $3,680 by misapplying the two-child EITC schedule to the household's earnings. The federal EITC is $7,316, and Louisiana refunds 5% of it, or $365.80." +us,scenario_038,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model contradicted its own recognition that the household was near the two-child EITC plateau by substituting a federal EITC of only $2,640. The computed federal EITC is $7,316, whose 5% Louisiana match is $365.80." +us,scenario_038,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model rounded the federal EITC base to about $7,000 instead of computing the exact $7,316 amount. Louisiana's 5% refundable match therefore equals $365.80 rather than $350." +us,scenario_038,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Louisiana eliminated its refundable EITC. Louisiana applies a refundable 5% match to this household's $7,316 federal EITC, yielding $365.80." +us,scenario_038,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly classified Louisiana's EITC as nonrefundable. It is refundable in this calculation and equals 5% of the $7,316 federal EITC, or $365.80." +us,scenario_038,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used an incorrect federal EITC base of $7,354. The exact federal EITC is $7,316, so the correctly applied 5% Louisiana rate produces $365.80 rather than $367.70." +us,scenario_038,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted Louisiana's refundable EITC entirely. The household's $7,316 federal EITC triggers a refundable Louisiana credit of 5%, equal to $365.80." +us,scenario_038,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model correctly used Louisiana's 5% rate but understated the federal EITC as $7,250. The correct $7,316 federal credit produces a state credit of $365.80." +us,scenario_038,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The answer implies a federal EITC base of $7,068.60 under Louisiana's 5% formula. The computed federal EITC is $7,316, so the state credit is $365.80." us,scenario_038,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly treated Louisiana's EITC as nonrefundable and tied it to positive state tax liability. The refundable credit equals 5% of the household's $7,316 federal EITC regardless of the pre-credit state liability, producing $365.80." -us,scenario_038,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to apply Louisiana's refundable EITC to this qualifying household. Five percent of the $7,316 federal EITC is $365.80." -us,scenario_038,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that income at this level produces no Louisiana refundable credit. The household qualifies for a $7,316 federal EITC and therefore receives Louisiana's 5% refundable EITC of $365.80." -us,scenario_038,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the absence of separately listed credit inputs as eliminating Louisiana's EITC. EITC eligibility follows from earned income, filing status, and two qualifying children, yielding a federal EITC of $7,316 and a Louisiana refundable credit of $365.80." -us,scenario_038,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a federal EITC of $8,231, exceeding the applicable amount for this household. The correct federal EITC is $7,316, and Louisiana's 5% rate produces $365.80 rather than $411.55." -us,scenario_038,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly used the $7,316 federal EITC but applied an obsolete or otherwise incorrect 3.5% Louisiana rate. The applicable rate is 5%, so the refundable state credit is $365.80." -us,scenario_038,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model failed to recognize that the household facts trigger Louisiana's refundable EITC. The $7,316 federal EITC generates a 5% state credit of $365.80." -us,scenario_038,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Louisiana has no applicable state EITC. This qualifying household receives a refundable Louisiana EITC equal to 5% of $7,316, or $365.80." -us,scenario_038,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model incorrectly required qualifying expenses or a separately supplied EITC match and omitted the credit derived from the household's earnings and children. Louisiana matches 5% of the calculated $7,316 federal EITC, producing $365.80." -us,scenario_038,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for state_refundable_credits, violating the required output contract." -us,scenario_038,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model correctly applied Louisiana's 5% rate but used an unsupported rounded federal EITC of $6,500. The applicable federal EITC is $7,316, so the state refundable credit is $365.80 rather than $325." -us,scenario_038,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model approximated the federal EITC with an estimated maximum of $7,430 instead of calculating the household's applicable $7,316 credit. At Louisiana's 5% rate, the exact refundable credit is $365.80 rather than $371.50." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model included $25,950 of estate income in AGI, which also inflated provisional income and taxable Social Security, then omitted the $3,319.81 qualified business income deduction. Its surviving-spouse recomputation therefore began from $73,707 instead of the traced $32,699.07 AGI." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model included estate income and overstated taxable Social Security, then invented an additional standard deduction for disability and enough unspecified elderly/disabled credits to erase the tax. Disability at age 61 does not create that standard-deduction addition, while the actual deductions leave $13,279.25 taxable and $1,345.51 due." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model put the $25,950 estate-income input into AGI, causing it to tax 85% of Social Security, and it omitted the $3,319.81 QBI deduction. The trace instead has $10,129.10 of taxable Social Security and $32,699.07 of AGI before the standard and QBI deductions." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model included estate income when calculating both AGI and Social Security provisional income, producing $25,186.63 of taxable benefits, and it omitted the QBI deduction. The correct computation uses $10,129.10 of taxable Social Security and subtracts $3,319.81 of QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI, taxed $30,689 of Social Security as a result, and failed to apply the $3,319.81 QBI deduction. Those errors raised taxable income from $13,279.25 to roughly $62,500." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated estate income as federal gross income, which inflated provisional income and taxable Social Security to $25,186.63, and it never applied the $3,319.81 QBI deduction. Its detailed medical-itemization analysis does not cure those two errors because the standard deduction is used." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model included the estate-income input in AGI, used an unsupported shortcut of taxing 85% of Social Security, and omitted the QBI deduction. The trace instead includes $10,129.10 of taxable Social Security and reduces income by a $3,319.81 QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model included estate income and 85% of Social Security in AGI, omitted the QBI deduction, and applied an obsolete pre-TCJA-style $8,800 deduction and 15% bracket schedule. The applicable computation uses a $16,100 standard deduction, a $3,319.81 QBI deduction, and the standard 2026 rate schedule on $13,279.25." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model included estate income, taxed 85% of Social Security, omitted the QBI deduction, and used an expired-law personal-exemption and 15%/25% rate structure. The traced 2026 computation instead taxes $13,279.25 after the $16,100 standard deduction and $3,319.81 QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its unexplained $5,354 estimate is inconsistent with the traced taxable income of $13,279.25 and reflects the same shortcut of aggregating the listed income and subtracting only a standard deduction. The computation excludes estate income from AGI, includes only $10,129.10 of taxable Social Security, and applies a $3,319.81 QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model explicitly used $79,209 of AGI by including estate income and 85% of Social Security, then omitted the QBI deduction. The traced AGI is $32,699.07 and falls to $13,279.25 after both deductions." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model included estate income, taxed 85% of Social Security, omitted the QBI deduction, and applied a post-expiration personal-exemption tax structure. The applicable calculation uses $32,699.07 of AGI and the $16,100 standard plus $3,319.81 QBI deductions." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model's $79,209.25 AGI wrongly includes estate income and $30,689.25 of taxable Social Security. It also omitted the $3,319.81 QBI deduction and used $15,700 instead of the traced $16,100 standard deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included estate income in provisional income and AGI, forcing taxable Social Security to its 85% cap, and omitted the QBI deduction. It also estimated the standard deduction at $15,432 rather than using $16,100." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that surviving-spouse or elderly treatment eliminated taxable income without computing the income stack. After excluding estate income from AGI and applying the standard and QBI deductions, taxable income remains $13,279.25 rather than zero." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invented unspecified nonrefundable credits sufficient to eliminate liability, despite no such credits appearing in the household facts or trace. The deductions leave $13,279.25 taxable, producing $1,345.51 before refundable credits." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model included estate income in non-Social-Security income, taxed 85% of Social Security, and omitted the QBI deduction. Consequently it used $79,209 instead of $32,699.07 of AGI and overstated taxable income by more than $33,000." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model included estate income when computing AGI and provisional income, producing $25,186.63 of taxable Social Security, and omitted the $3,319.81 QBI deduction. The trace instead uses $10,129.10 of taxable benefits and $13,279.25 of taxable income." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's $73,707 AGI includes estate income and an overstated $25,187 taxable Social Security amount. It also omitted the $3,319.81 QBI deduction, whereas the traced computation reaches $13,279.25 of taxable income." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model included estate income and overstated taxable Social Security in its $79,209 AGI, omitted the QBI deduction, and invented age-61 and senior deductions. Age 61 supplies no federal aged additional deduction or senior deduction, and the traced deductions total $19,419.81." -us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The unexplained $8,000 estimate is consistent with treating estate income and a large portion of Social Security as taxable while subtracting only ordinary deductions. The traced calculation excludes estate income from AGI, taxes only $10,129.10 of Social Security, and applies a $3,319.81 QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model included estate income, taxed 85% of Social Security, omitted the QBI deduction, and applied an expired-law personal exemption with 15% and 25% brackets. The applicable 2026 calculation uses $32,699.07 of AGI and $13,279.25 of taxable income under the standard rate schedule." -us,scenario_039,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI, taxed Social Security at the 85% cap, and omitted the QBI deduction. It also used an estimated $15,420 standard deduction instead of $16,100, producing $63,789 rather than $13,279.25 of taxable income." +us,scenario_038,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to apply Louisiana's refundable EITC to an eligible federal EITC recipient. The 5% match on $7,316 equals $365.80." +us,scenario_038,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable Louisiana credit was available at this income. The household receives a $7,316 federal EITC and therefore a $365.80 refundable Louisiana EITC." +us,scenario_038,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the absence of separately listed credit inputs as excluding Louisiana's EITC. EITC eligibility follows from the household's earnings and qualifying children, and Louisiana refunds 5% of the $7,316 federal credit, or $365.80." +us,scenario_038,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model inflated the federal EITC to $8,231 instead of applying the two-child 2026 EITC computation that yields $7,316. At Louisiana's 5% rate, the state credit is $365.80 rather than $411.55." +us,scenario_038,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used a 3.5% Louisiana EITC rate instead of the applicable 5% rate. Applying 5% to the correctly identified $7,316 federal EITC yields $365.80." +us,scenario_038,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model failed to recognize that the household's federal EITC triggers Louisiana's refundable EITC. The state credit is 5% of $7,316, or $365.80." +us,scenario_038,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model incorrectly stated that Louisiana has no applicable state EITC or other refundable offset. Louisiana refunds 5% of this household's $7,316 federal EITC, producing $365.80." +us,scenario_038,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model incorrectly concluded that the household had no EITC match because no qualifying expense was listed. Louisiana's refundable EITC requires the federal EITC rather than a separate expense, and 5% of $7,316 is $365.80." +us,scenario_038,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model used an estimated federal EITC of $7,422 rather than the computed $7,316. The correct federal base multiplied by Louisiana's 5% rate gives $365.80, not $371." +us,scenario_038,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, violating the required output contract." +us,scenario_038,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model understated the federal EITC as $6,500 instead of computing $7,316 under the two-child schedule. Louisiana's 5% match is therefore $365.80 rather than $325." +us,scenario_038,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model approximated the household's federal EITC at the maximum of $7,430 instead of calculating the exact $7,316 credit from the applicable earnings measure and schedule. The 5% Louisiana credit is $365.80, not $371.50." +us,scenario_038,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model omitted Louisiana's refundable EITC despite the household qualifying for a federal EITC. Louisiana refunds 5% of the $7,316 federal credit, yielding $365.80." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model included the $25,950 estate-income input in AGI, used qualifying-surviving-spouse Social Security thresholds and joint deductions, and omitted the $3,319.81 QBI deduction. The computation instead uses $10,129.10 of taxable Social Security and the $16,100 single standard deduction, producing taxable income of $13,279.25." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model included estate income and 85% of Social Security in AGI, then invented a disability-based additional standard deduction and enough unspecified nonrefundable relief to erase the tax. Disability at age 61 does not itself create that standard-deduction addition, and the actual deductions leave $13,279.25 taxable." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Although it correctly rejected qualifying-surviving-spouse filing status without a dependent child, it included the $25,950 estate-income input in AGI, taxed 85% of Social Security, and omitted the $3,319.81 QBI deduction. Those errors raised taxable income from $13,279.25 to its estimated $59,459." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model treated the filer as a qualifying surviving spouse and therefore used joint Social Security thresholds, a joint standard deduction, and joint brackets. The computation uses single-filer treatment, excludes the estate-income input from AGI, and applies a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model included the $25,950 estate-income input, assumed 85% of Social Security was taxable, and omitted the $3,319.81 QBI deduction. The trace instead has AGI of $32,699.07 and taxable income of $13,279.25." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model assigned qualifying-surviving-spouse status and used joint Social Security thresholds, a joint standard deduction, and joint brackets. It also included estate income and missed the $3,319.81 QBI deduction; the applicable computation uses the $16,100 single standard deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model included estate income, taxed 85% of Social Security, and used a projected joint standard deduction while omitting the QBI deduction. The correct AGI is $32,699.07, followed by $19,419.81 of total deductions." +us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly applied an $8,800 post-expiration single deduction and pre-TCJA 10%/15% rates, while also including estate income and 85% of Social Security. The applicable 2026 computation uses the $16,100 standard deduction, the current rate schedule, and a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied an obsolete post-TCJA-expiration structure with a personal exemption and 15%/25% brackets. It also included estate income, taxed 85% of Social Security, and omitted the QBI deduction, rather than taxing $13,279.25 under the applicable schedule." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its unsupported $5,354 estimate reflects a generic aggregation of Social Security, retirement, pension, and estate income less a standard deduction. The required derivation uses only $10,129.10 of taxable Social Security, excludes the estate-income input from AGI, and subtracts the $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model explicitly used AGI of about $79,209, which includes estate income and 85% of Social Security. It missed the traced AGI of $32,699.07 and the $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model included estate income and 85% of Social Security, then applied an obsolete personal exemption and post-expiration rate structure. The applicable calculation has $13,279.25 of taxable income after the standard and QBI deductions." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model's $79,209.25 AGI improperly includes the estate-income input and $30,689.25 of taxable Social Security. It also omitted the $3,319.81 QBI deduction and used the wrong standard-deduction amount." +us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included estate income and applied the 85% Social Security cap under its single-filer provisional-income calculation. It then omitted the $3,319.81 QBI deduction, inflating taxable income from $13,279.25 to $63,777.25." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that surviving-spouse or elderly treatment reduced taxable income to zero. The traced standard and QBI deductions leave $13,279.25 taxable, generating $1,345.51 before refundable credits." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and unspecified nonrefundable credits eliminated the liability without identifying any such credit. The computation leaves $13,279.25 taxable and no nonrefundable credit reduces the resulting $1,345.51." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model used surviving-spouse joint brackets and a $32,200 joint standard deduction, while including estate income and 85% of Social Security. The computation uses single-filer treatment, $10,129.10 of taxable Social Security, and the $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-surviving-spouse joint treatment and therefore used joint Social Security thresholds, deductions, and brackets. It also included estate income and omitted the QBI deduction that reduces traced taxable income to $13,279.25." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model treated the filer as qualifying surviving spouse and used joint Social Security thresholds, a $32,200 joint standard deduction, and joint brackets. The applicable computation instead uses the $16,100 single deduction and a separate $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model included estate income and then invented both an age-61 additional deduction and a senior deduction. Neither deduction applies at age 61, while the actual additional deduction is the $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its unexplained $8,000 estimate is consistent with broadly taxing estate, retirement, pension, and Social Security income before generic standard and medical deductions. The trace instead produces AGI of $32,699.07 and applies the standard plus QBI deductions to reach $13,279.25 taxable." +us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model included estate income and 85% of Social Security, then used an obsolete post-TCJA-expiration deduction, personal exemption, and 15%/25% brackets. The applicable calculation uses the $16,100 standard deduction, a $3,319.81 QBI deduction, and the current rate schedule." +us,scenario_039,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model's $79,209 AGI includes estate income and $30,689 of taxable Social Security. It also omitted the $3,319.81 QBI deduction, causing it to tax $63,789 instead of $13,279.25." +us,scenario_039,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model correctly used the $16,100 standard deduction but began from an inflated $79,209 AGI by including estate income and 85% of Social Security. It also omitted the $3,319.81 QBI deduction." us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output. -us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI and provisional income, causing $30,689.25 of Social Security to be taxable, and omitted the $3,319.81 QBI deduction. Its bracket arithmetic is internally coherent only for the overstated $63,109.25 taxable-income base." -us,scenario_039,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model double counted Social Security—first in its gross-income total and again as taxable Social Security—while also including estate income and inventing age-61 and disability standard-deduction additions. It omitted the QBI deduction and consequently estimated taxable income near $79,214 instead of $13,279.25." -us,scenario_039,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"After several contradictory filing-status and Social Security calculations, the model's final path still included estate income in provisional income and AGI, producing about $25,197 of taxable Social Security, and omitted the $3,319.81 QBI deduction. It also invented mortgage interest and SALT amounts from unlisted facts during its abandoned itemization analysis." +us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model included estate income and treated $30,689.25, or 85% of benefits, as taxable Social Security. Although it used the correct $16,100 standard deduction, it omitted the $3,319.81 QBI deduction and inflated taxable income to $63,109.25." +us,scenario_039,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model double-counted Social Security by first including the full $36,105 in gross income and then adding its taxable portion again. It also invented age-61 and disability standard-deduction additions, included estate income, and omitted the $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"After several contradictory filing-status and Social Security calculations, the model settled on joint treatment, included estate income, and omitted the QBI deduction. Its taxable Social Security arithmetic was also off by $10; the trace uses $10,129.10 of taxable benefits and $13,279.25 of taxable income." +us,scenario_039,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model deducted only half the self-employment loss, included estate income and 85% of Social Security, and invented a credit for other dependents based on pension income. Pension income does not create an other-dependent credit, and the full $6,260.03 loss plus the $3,319.81 QBI deduction enters the traced computation." us,scenario_039,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_039,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly placed the head into Virginia's expansion adult Medicaid pathway while also claiming MAGI was within the 138% FPL limit. PolicyEngine's trace gives MAGI at 3.68 times FPL and medicaid_category = NONE, so the disabled status did not create an ABD or expansion category that overrides the failed income/categorical screen." -us,scenario_039,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the unlisted disability fact as automatic Medicaid eligibility and skipped Virginia's categorical and income pathway tests. PolicyEngine finds no Medicaid category for the head, SSI received is zero, and the MAGI level is 3.68 times FPL, so disability alone does not make the head eligible." -us,scenario_039,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated disability plus Social Security retirement income as proof of the under-65 Medicare pathway. Retirement benefits are not SSDI, and the facts do not establish SSDI entitlement for 24 months or another qualifying condition." -us,scenario_039,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model conflated the disability flag and Social Security retirement income with receipt of Social Security disability benefits. At age 61, Medicare requires the separate under-65 entitlement conditions, including the applicable SSDI waiting period, which are not present." -us,scenario_039,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model cited the 24-month SSDI rule but assumed it was satisfied from disability alone. No SSDI receipt or 24-month entitlement period is listed, so the under-65 disability pathway is not established." -us,scenario_039,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model converted a general disability status directly into Medicare eligibility. Disability alone does not establish the required under-65 Medicare entitlement pathway, and the head is below age 65." -us,scenario_039,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model incorrectly asserted that PolicyEngine treats every disabled person as Medicare eligible. The head is 61, and the inputs do not establish SSDI entitlement for the required period or another under-65 qualifying condition." -us,scenario_039,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,The model stated the 24-month SSDI requirement but replaced it with the bare disability flag; it also invoked surviving-spouse status without facts establishing a Medicare entitlement pathway. Neither disability nor surviving-spouse status alone makes this 61-year-old eligible. -us,scenario_039,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model treated age 61 plus disability as sufficient for Medicare. Because the head is under 65 and no qualifying SSDI entitlement period or other special pathway is listed, those facts yield ineligibility." -us,scenario_039,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model assumed the generic disabled input was itself a qualifying Medicare condition. Under-65 eligibility requires additional entitlement facts, such as SSDI entitlement for the required waiting period, which the household does not have." -us,scenario_039,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model assumed coverage begins upon disability under unspecified qualifying rules. Medicare does not begin from the disability flag alone, and no SSDI entitlement duration or other qualifying under-65 condition is provided." -us,scenario_039,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,The model treated disability as automatically overcoming the age-65 threshold. The facts establish neither the required SSDI-based entitlement nor another special Medicare pathway for this 61-year-old. -us,scenario_039,head_medicare_eligible,grok-4.3,llm_error,age_disability,False,"The model equated being disabled with being Medicare eligible. A 61-year-old needs a qualifying under-65 entitlement pathway, and none is listed." -us,scenario_039,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model reduced the rule to age 65 or disabled, omitting the additional conditions on under-65 disability-based Medicare eligibility. The disability flag does not establish SSDI entitlement for the required period." -us,scenario_039,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model correctly mentioned the 24-month SSDI condition but assumed it from the statement that the head is disabled. SSDI receipt and 24 months of entitlement are unlisted and false, so the disability pathway is not satisfied." -us,scenario_039,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model applied Social Security and Medicare employee payroll tax rates to an inferred wage base even though the household facts list no wage or salary income and instruct that unlisted numeric inputs are zero. It treated unearned income and retirement distributions as if they generated employee FICA withholding, instead of limiting payroll_tax to employee-side taxes on listed wages." -us,scenario_039,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable payroll_tax value. The correct derivation yields zero because there are no listed wages or salary earnings subject to employee Social Security or Medicare tax, and the prompt requires missing numeric inputs to be treated as zero." +us,scenario_039,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly concluded that the head's income was below the expansion threshold and treated disability as compatible with an eligible adult or ABD pathway despite acknowledging the resource-limit problem. The trace places MAGI at 3.68 times FPL, so the expansion pathway fails, and the head qualifies for no ABD or other Medicaid category." +us,scenario_039,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated disability as sufficient for Medicaid and failed to apply Virginia's categorical and income tests. The head has no qualifying Medicaid category, and the MAGI expansion pathway fails at 3.68 times FPL." +us,scenario_039,head_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated disability plus Social Security retirement income as proof of disability-based Medicare entitlement. Retirement benefits are not SSDI, and the facts establish neither SSDI entitlement for the required waiting period nor another under-65 Medicare pathway." +us,scenario_039,head_medicare_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model conflated the generic disability flag and Social Security retirement income with receipt of Social Security disability benefits. At age 61, Medicare requires a qualifying under-65 pathway, and neither SSDI entitlement for the required waiting period nor ESRD is present." +us,scenario_039,head_medicare_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model cited the 24-month SSDI rule but assumed that being listed as disabled satisfies it. The household facts do not list SSDI receipt or completion of the waiting period, so the disability pathway is not established." +us,scenario_039,head_medicare_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model converted a general disability status directly into Medicare eligibility. Disability alone is insufficient before age 65; the facts provide no qualifying SSDI entitlement, completed waiting period, or ESRD pathway." +us,scenario_039,head_medicare_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that PolicyEngine treats every disabled person as Medicare eligible. The head is under 65, and the generic disability input does not establish the separate SSDI-duration or ESRD conditions required for Medicare." +us,scenario_039,head_medicare_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model correctly named the 24-month SSDI condition but replaced it with the bare disability flag; no SSDI receipt or duration is listed. It also invoked surviving-spouse status even though that status alone does not create Medicare eligibility for a 61-year-old. +us,scenario_039,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model treated age 61 plus disability as sufficient without applying the qualifying-entitlement requirement. The facts establish no SSDI waiting-period completion or other under-65 Medicare pathway. +us,scenario_039,head_medicare_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model assumed that the household's generic disability flag is itself a qualifying Medicare condition under PolicyEngine. Before age 65, disability-based eligibility requires the specified Medicare entitlement pathway, which the supplied facts do not establish." +us,scenario_039,head_medicare_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed coverage begins from disability status without checking whether the qualifying disability rules were met. No SSDI entitlement and waiting-period completion or other under-65 pathway is listed, so age 61 does not qualify." +us,scenario_039,head_medicare_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model equated disability with automatic Medicare qualification despite the head being under 65. The generic disability fact does not supply the required SSDI entitlement history or another qualifying Medicare condition. +us,scenario_039,head_medicare_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model applied an automatic disabled-equals-eligible shortcut. Medicare eligibility at age 61 requires a qualifying under-65 entitlement pathway, and none is included in the household facts." +us,scenario_039,head_medicare_eligible,kimi-k3,llm_error,categorical_eligibility,False,The model misstated the rule as age 65 or any disability. Disability alone does not establish Medicare eligibility before 65; the necessary SSDI-duration or other qualifying condition is absent. +us,scenario_039,head_medicare_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model named the 24-month SSDI requirement but inferred that it was satisfied solely because the head is disabled. The prompt lists Social Security retirement income, not SSDI or its duration, and unlisted qualifying facts are false." +us,scenario_039,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented wage earnings and applied the 7.65% employee Social Security and Medicare rate to an unstated payroll-tax base. The household has no listed wage or salary income, while its listed estate, retirement, IRA, pension, and negative self-employment income are not subject to employee payroll tax, yielding $0." +us,scenario_039,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract." us,scenario_039,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $25,950 estate-income input in federal and Virginia AGI, yielding $48,520 before the Social Security subtraction instead of $22,569.97 of Virginia AGI. It also omitted the $930 personal exemption, so its taxable-income base was overstated further." -us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model included estate income in the tax base and then invented retirement and disability exclusions or credits that eliminate the liability. The actual deductions are the $10,129.10 Social Security subtraction, $8,750 standard deduction, and $930 personal exemption, leaving $12,889.97 taxable and $514.50 of tax." -us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model included the $25,950 estate-income input in Virginia AGI and incorrectly treated surviving-spouse status as entitling this single taxpayer to the joint Virginia standard deduction. Virginia applies the $8,750 deduction and $930 exemption to $22,569.97 of Virginia AGI, producing $514.50." -us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly carried the $25,950 estate-income input into Virginia AGI, using $48,520 instead of $22,569.97. It also invented an additional disabled exemption and an unexplained adjustment rather than applying the traced $8,750 standard deduction and $930 personal exemption." -us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Its roughly $40,000 taxable-income estimate includes the $25,950 estate-income input in Virginia AGI. Excluding that amount and applying the $8,750 standard deduction and $930 exemption leaves $12,889.97 taxable, not about $40,000." -us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated all $36,105 of Social Security and the $25,950 estate-income input as federal AGI, then invented a $20,000 subtraction for disabled taxpayers and joint-filer deductions and exemptions. The trace instead has $32,699.07 federal AGI, a $10,129.10 taxable-Social-Security subtraction, and single deductions totaling $9,680." -us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly removed federally taxable Social Security but incorrectly retained the $25,950 estate-income input, producing $48,520 rather than $22,569.97 of Virginia AGI. Its final $3,400 also contradicts its own bracket calculation of about $2,044." -us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used federal AGI without applying Virginia's $10,129.10 subtraction for taxable Social Security and also retained the $25,950 estate-income input in AGI. Those errors raised taxable income from $12,889.97 to $69,529." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly subtracted taxable Social Security but incorrectly included the $25,950 estate-income input, leaving $48,520 of Virginia AGI. The traced Virginia AGI is $22,569.97, and the applicable standard deduction is $8,750 rather than $8,500." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $2,420 implies a taxable-income base far above the traced $12,889.97. The model failed to derive Virginia AGI of $22,569.97 by excluding estate income and subtracting taxable Social Security before applying the $8,750 deduction and $930 exemption." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $1,990 answer matches the shortcut that subtracts taxable Social Security but retains the $25,950 estate-income input, leaving about $48,520 of Virginia AGI. Excluding estate income yields $22,569.97 of Virginia AGI and $514.50 of tax." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly retained the $25,950 estate-income input after subtracting Social Security, overstating Virginia AGI as $48,520. It also used incorrect $9,000 and $800 deduction amounts instead of the traced $8,750 standard deduction and $930 exemption." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly included estate income in the $48,520 Virginia AGI base and used an $8,000 standard deduction. Excluding estate income and using the $8,750 deduction plus $930 exemption produces $12,889.97 of taxable income." -us,scenario_039,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model's Social Security calculation and bracket arithmetic are coherent, but it incorrectly included the $25,950 estate-income input in non-Social-Security AGI. Removing it changes Virginia AGI from $48,520 to $22,569.97 and tax from $1,990.18 to $514.50." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that age-based treatment eliminated the taxable base even though age 61 supplies no such Virginia deduction in the trace. After the Social Security subtraction, standard deduction, and personal exemption, taxable income remains $12,889.97 and generates $514.50 of tax." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model incorrectly classified the household as a zero-tax low-income case. Virginia taxable income is $12,889.97 after the traced adjustments, which lies within the positive 2%, 3%, and 5% brackets and produces $514.50." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $25,950 estate-income input in Virginia AGI, leaving $48,520 after the Social Security subtraction. The correct Virginia AGI is $22,569.97, followed by an $8,750 standard deduction and $930 exemption." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model applied a surviving-spouse or joint standard deduction to an income base that retained the $25,950 estate-income input. The traced computation uses $22,569.97 of Virginia AGI and the $8,750 standard deduction for this taxpayer, yielding $514.50." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model incorrectly used a $17,500 joint standard deduction based on surviving-spouse status and retained estate income in the Virginia AGI base. The calculation uses a single $8,750 deduction against $22,569.97 of Virginia AGI." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The answer reflects a surviving-spouse deduction applied to an overstated income base that includes the $25,950 estate-income input. Virginia taxable income is instead $12,889.97 after the traced $8,750 deduction and $930 exemption." -us,scenario_039,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that deductions and exemptions fully offset income. They reduce Virginia AGI of $22,569.97 to positive taxable income of $12,889.97, which produces $514.50 under the Virginia brackets." -us,scenario_039,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model correctly removed taxable Social Security but incorrectly retained the $25,950 estate-income input, producing $48,520 of Virginia AGI. It also used an $8,000 standard deduction instead of $8,750." -us,scenario_039,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $25,950 estate-income input in Virginia AGI and used an $8,000 standard deduction. The trace yields $22,569.97 of Virginia AGI and a $12,889.97 taxable base after the $8,750 deduction and $930 exemption." -us,scenario_039,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric value or explanation for the requested output, so it failed the required output contract." -us,scenario_039,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the correct deduction and exemption amounts but incorrectly retained the $25,950 estate-income input, leaving $48,520 of Virginia AGI. Excluding that input yields the traced $22,569.97 Virginia AGI and $514.50 tax." -us,scenario_039,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the full $84,625 household-income total as federal AGI, retained estate income and gross Social Security, and invented age-65 and disability deductions despite the taxpayer being 61. Its submitted $1,500 also does not follow from its own stated $4,100 tax computation." -us,scenario_039,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated gross Social Security and the $25,950 estate-income input as federal AGI, mislabeled the $930 personal exemption as the standard deduction, and omitted the actual $8,750 standard deduction. Its submitted $3,260.45 also contradicts its own computed $1,878.93." +us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model included the $25,950 estate-income input in federal and Virginia AGI, producing a post-Social-Security Virginia base near $48,520 instead of $22,569.97. Applying the $8,750 deduction and $930 exemption to the correct base yields $12,889.97 of taxable income and $514.50 of tax." +us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented a Virginia retirement-income credit and disability exclusion that eliminate the tax, while also using a federal-sized standard deduction. The applicable computation uses Virginia AGI of $22,569.97, the $8,750 Virginia standard deduction, and the $930 exemption, leaving $12,889.97 taxable." +us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model treated surviving-spouse status as entitling this single Virginia filer to the roughly $17,000 joint standard deduction and included estate income in the Virginia base. The trace applies the $8,750 standard deduction to Virginia AGI of $22,569.97, followed by the $930 exemption." +us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model included the $25,950 estate-income input in Virginia AGI, raising the post-Social-Security base to $48,520 rather than $22,569.97. It then introduced unsupported disability and medical adjustments instead of applying the traced $8,750 standard deduction and $930 exemption." +us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model’s roughly $40,000 taxable-income estimate comes from retaining the $25,950 estate-income input in the Virginia income base. Correct Virginia taxable income is $12,889.97 after the Social Security subtraction, $8,750 standard deduction, and $930 exemption." +us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated all $36,105 of Social Security and the $25,950 estate-income input as federal AGI, then fabricated a $20,000 subtraction for disabled taxpayers’ IRA and pension income and a joint deduction. The trace instead starts at federal AGI of $32,699.07, subtracts $10,129.10, and applies only the $8,750 deduction and $930 exemption shown." +us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model included the $25,950 estate-income input in the post-Social-Security Virginia base, calculating $48,520 rather than $22,569.97. Its final $3,400 also contradicts its own bracket calculation of about $2,044 and has no supporting computation." +us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model taxed federal AGI of $79,209 without applying Virginia’s $10,129.10 subtraction, primarily for taxable Social Security, and that AGI itself improperly included the estate-income input. Virginia AGI is $22,569.97 before the $8,750 deduction and $930 exemption." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly removed federally taxable Social Security from its assumed AGI but still retained the $25,950 estate-income input, leaving $48,520 instead of $22,569.97. That single inclusion inflated taxable income from $12,889.97 to $39,090." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $2,420 answer implies a taxable-income base far above the traced $12,889.97. The correct sequence is $32,699.07 federal AGI minus $10,129.10 of Virginia subtractions, then minus the $8,750 standard deduction and $930 exemption." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The $1,990 result matches the common shortcut of leaving $48,520 after subtracting taxable Social Security, which retains the $25,950 estate-income input in Virginia AGI. The correct post-subtraction Virginia AGI is $22,569.97." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model retained the $25,950 estate-income input after subtracting taxable Social Security, producing Virginia AGI of $48,520 instead of $22,569.97. It also used incorrect $9,000 and $800 deduction and exemption amounts rather than $8,750 and $930." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model’s federal AGI of $79,209 improperly includes the estate-income input, so its post-Social-Security base remains $48,520 rather than $22,569.97. It additionally used an $8,000 standard deduction instead of the traced $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model’s detailed computation wrongly includes the $25,950 estate-income input among AGI components, making non-Social-Security income $48,520. Removing that inclusion gives the traced Virginia AGI of $22,569.97 after the $10,129.10 subtraction." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that age-based treatment eliminates the taxable base, but age 61 does not produce the deduction needed for that result. Virginia taxable income remains $12,889.97 after the traced subtraction, standard deduction, and exemption." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model treated the household as a zero-tax low-income case without computing Virginia taxable income or an actual nonrefundable credit. The traced taxable income is $12,889.97, and the bracket calculation leaves $514.50 before refundable credits." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model retained the $25,950 estate-income input in its $48,520 Virginia AGI estimate. The traced post-subtraction Virginia AGI is $22,569.97, so taxable income is $12,889.97 rather than $39,090." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model applied a surviving-spouse or joint deduction framework and still produced tax consistent with an inflated income base. The trace treats the filer with an $8,750 standard deduction and $930 exemption applied to Virginia AGI of $22,569.97." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model incorrectly used a $17,500 joint standard deduction based on surviving-spouse status and began from the inflated $48,520 base that includes estate income. The traced deduction is $8,750, but it applies to the much lower Virginia AGI of $22,569.97." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model’s $1,789 estimate implies that it retained estate income in the Virginia base despite mentioning the Social Security subtraction. The correct subtraction produces Virginia AGI of $22,569.97 and taxable income of $12,889.97." +us,scenario_039,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that deductions and exemptions fully offset Virginia income. They reduce Virginia AGI of $22,569.97 by $9,680, leaving $12,889.97 taxable and $514.50 of tax." +us,scenario_039,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model retained the $25,950 estate-income input in Virginia AGI, leaving $48,520 after the Social Security subtraction instead of $22,569.97. It also used an $8,000 standard deduction rather than $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model retained the $25,950 estate-income input in Virginia AGI, leaving $48,520 after the Social Security subtraction instead of $22,569.97. It also understated the standard deduction at $8,000 rather than $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model included the $25,950 estate-income input in federal and Virginia AGI, producing a $48,520 post-Social-Security base rather than $22,569.97. It then used an $8,000 approximate deduction instead of the traced $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so the required structured answer was missing." +us,scenario_039,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model’s arithmetic is internally consistent after its starting point, but that starting point improperly retains the $25,950 estate-income input in Virginia AGI. Virginia AGI is $22,569.97, not $48,520, before the correctly identified $8,750 deduction and $930 exemption." +us,scenario_039,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the full $84,625 household-income total as Virginia taxable income and invented age-65 and disability deductions despite the taxpayer being 61. Its submitted $1,500 then abandoned its own roughly $4,100 calculation instead of following the traced AGI and bracket computation." +us,scenario_039,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated total household income, including all Social Security and the estate-income input, as federal AGI and confused the $930 personal exemption with the standard deduction. Its submitted $3,260.45 also contradicts its own stated $1,878.93 bracket result." +us,scenario_039,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model invented an $11,898 age-61 exemption and treated $2,030 of pension income as a dollar-for-dollar nonrefundable credit. Pension income is not such a credit; the traced deductions leave $12,889.97 taxable and $514.50 due." us,scenario_039,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_040,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applied a post-TCJA-sunset structure with personal exemptions and a too-small standard deduction, leaving artificial taxable income of $2,850. Under the PolicyEngine 2026 calculation, the married-filing-jointly standard deduction with age additions exceeds the $32,300 AGI, so taxable income is $0." -us,scenario_040,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model treated TCJA expiration as creating a lower standard deduction plus personal exemptions and subtracted only $30,300 from AGI. PolicyEngine's 2026 current-law deduction calculation wipes out the full $32,300 AGI, so the 10% bracket is never reached." -us,scenario_040,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used estimated post-TCJA deduction and exemption amounts that left $3,100 of taxable income. The correct PolicyEngine computation applies deductions large enough for this elderly joint-filing unit to reduce taxable income to $0." -us,scenario_040,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model combined a lower estimated standard deduction with personal exemptions and left $2,200 subject to the 10% bracket. PolicyEngine's 2026 standard deduction treatment for this married elderly couple eliminates the entire $32,300 AGI before tax rates apply." -us,scenario_040,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model inflated pre-TCJA standard deduction and personal exemption amounts from old-law baselines and produced $1,748 of taxable income. PolicyEngine applies the current-law 2026 deduction structure, which reduces taxable income to zero before any bracket tax is computed." -us,scenario_040,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable value or supporting calculation for federal_income_tax_before_refundable_credits. This is a missing-output failure rather than a substantive tax computation. -us,scenario_040,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly included workers' compensation in provisional income for Social Security taxation, raising taxable Social Security and AGI above PolicyEngine's $32,300 figure. It then applied a deduction amount that would eliminate the correct AGI but not its inflated AGI, creating $1,800 of artificial taxable income and $180 of tax." +us,scenario_040,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete TCJA-sunset regime with personal exemptions and only $18,850 of standard and age-based deductions. The applicable 2026 standard deduction and additional deductions for two spouses over 65 exceed the $32,300 AGI, so taxable income is zero rather than $2,850." +us,scenario_040,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model assumed the TCJA expired and combined personal exemptions with an understated $19,700 standard deduction and elderly add-ons. Under the applicable 2026 deduction rules, total standard and age-based deductions exceed $32,300 of AGI, leaving no taxable income to tax at 10%." +us,scenario_040,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used estimated post-TCJA personal exemptions and understated the married-joint standard deduction with two over-65 additions. The correct deduction exceeds the household's $32,300 AGI, so its asserted $3,100 of taxable income does not arise." +us,scenario_040,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model substituted personal exemptions and an understated $19,500 standard deduction for the applicable 2026 married-joint standard and age-based deductions. Those deductions eliminate the entire $32,300 AGI, rather than leaving $2,200 taxable at 10%." +us,scenario_040,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model inflation-adjusted pre-TCJA deductions and personal exemptions to construct only $30,552 of offsets. The applicable 2026 standard deduction plus both over-65 additions exceeds the $32,300 AGI, eliminating the $1,748 of taxable income on which it calculated $175." +us,scenario_040,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so it failed the required structured-output contract." +us,scenario_040,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the spouse's $3,200 workers' compensation in provisional income, overstating taxable Social Security from $4,300 to $8,450 and AGI from $32,300 to $36,450. With the trace-derived $32,300 AGI, the applicable married-joint standard deduction and two over-65 additions leave zero taxable income." +us,scenario_040,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated 85% of all Social Security benefits as taxable instead of applying the provisional-income formula, overstating taxable Social Security from $4,300 to $21,420 and AGI from $32,300 to $49,420. It also treated a senior deduction as a direct $650 tax subtraction; the correct AGI is fully absorbed by the applicable standard and age-based deductions before tax is calculated." us,scenario_040,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_040,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented an Arizona aged/disabled or spend-down eligibility pathway for the head even though the head is non-disabled, receives no SSI, and PolicyEngine assigns medicaid_category NONE. It also conflated the disabled spouse's status and possible ABD concepts with the head's own eligibility, then treated disregards and asset assumptions as creating eligibility despite the household's 2.46 FPL MAGI and no applicable category." -us,scenario_040,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model computed the correct wage-only FICA amount of $2,142 several times, then overrode it with $2,346 for invented state or comprehensive payroll obligations. Arizona has no mandatory employee payroll tax included in this output, and Additional Medicare Tax is not triggered at $28,000 of wages." -us,scenario_040,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model stated the correct Social Security component of $1,736 and Medicare component of $406 but submitted $1,948 instead of adding them. The missed computation step is the final summation of the two employee-side FICA components, which equals $2,142." -us,scenario_040,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model identified the correct tax bases and rates, then replaced the resulting $1,736 Social Security plus $406 Medicare calculation with an unsupported simplified estimate of $1,550. Employee payroll tax is not netted down for employer-side exclusions; those exclusions only keep employer taxes out of the output." -us,scenario_040,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"The model used the right base and combined 7.65% FICA rate but made an arithmetic error. Applying 7.65% to $28,000 of wages gives $2,142, not $2,149." -us,scenario_040,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable payroll_tax value or explanation. It failed the output contract rather than making a substantive tax-rule calculation. +us,scenario_040,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model improperly transferred the spouse’s disability to the nondisabled head and invoked Freedom to Work, disabled-adult-child, and spend-down pathways without the categorical facts required for any of them. The head is age 73, nondisabled, receives no SSI, and has income above the applicable aged pathway limit, so the engine assigns Medicaid category NONE." +us,scenario_040,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly derived $2,142, then overrode its own calculation with an unsupported $204 addition for unspecified state or federal obligations. Arizona imposes no included employee payroll tax here, and the household is far below the Additional Medicare Tax threshold." +us,scenario_040,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model explicitly calculated $1,736 of Social Security tax and $406 of Medicare tax but submitted $1,948 instead of summing them to $2,142. This is a final arithmetic or output-transcription error." +us,scenario_040,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model identified the correct components—$1,736 of Social Security tax and $406 of Medicare tax—but replaced their $2,142 sum with an unexplained $1,550 simplified estimate. Employee-side payroll tax includes both full components on all $28,000 of wages." +us,scenario_040,payroll_tax,grok-4.3,llm_error,other,False,"The model invoked the correct 7.65% combined employee rate but multiplied or rounded incorrectly: $28,000 × 7.65% equals exactly $2,142, not $2,149." +us,scenario_040,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required structured-output contract." us,scenario_040,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_040,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly bypassed the SNAP income eligibility screen by treating the elderly/disabled exception from the gross income test as enough to continue to a positive allotment. It then converted mortgage interest and real estate taxes into a large uncapped shelter deduction and estimated a minimum-like benefit, but the reference calculation stops at $0 because the two-person household's $56,400 annual income exceeds the applicable SNAP eligibility threshold." -us,scenario_040,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model converted age 69 and disability into Medicaid eligibility even after identifying that Arizona's aged/disabled pathway has income and resource tests. It then relied on speculative asset exclusions and favorable treatment of workers' compensation instead of applying the engine's pathway result: the spouse has no qualifying medicaid_category and MAGI income of 2.46 times FPL, so the eligibility output is 0." -us,scenario_040,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model assessed the spouse as if only the spouse's own $11,200 of wages and workers' compensation mattered, while dismissing the household and tax-unit income that leaves MAGI at 2.46 times FPL. It also misstated the disability pathway by saying the 69-year-old spouse is under 65, then treated categorical aged/disabled status as sufficient even though PolicyEngine assigns medicaid_category NONE." -us,scenario_040,spouse_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated ABD status as an eligibility pathway that overrides the actual income-category result, and it counted only the spouse's $8,000 of wages while excluding workers' compensation and the head's Social Security from the relevant eligibility picture. PolicyEngine's computation has the spouse in no Medicaid category, with MAGI income at 2.46 times FPL and no SSI receipt, so the spouse is not Medicaid eligible." -us,scenario_040,spouse_medicare_eligible,grok-build-0.1,llm_error,age_disability,False,"The model misread the spouse's age threshold, stating that age 69 is under 65. It then analyzed the disability-based Medicare pathway and SSDI waiting-period rule even though the spouse qualifies through the ordinary age-65 Medicare pathway." -us,scenario_040,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model correctly identified that Arizona subtracts Social Security and that the married-filing-jointly standard deduction exceeds the remaining $28,000 of wages, then abandoned that result and taxed an invented residual taxable base. With Arizona taxable income reduced to zero before credits, the 2.5% tax produces $0, not a $148 pre-refundable liability." -us,scenario_040,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model compressed the Arizona calculation into a generic 'small amount' estimate and failed to carry the standard-deduction calculation to zero taxable income. Arizona's Social Security subtraction leaves only wage income, and the joint standard deduction eliminates that base, so no pre-refundable Arizona tax remains." -us,scenario_040,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable value or explanation for this output. The required Arizona computation yields zero because Social Security is excluded and the joint standard deduction eliminates the remaining state taxable income. -us,scenario_040,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model understated Arizona's married-filing-jointly standard deduction by using about $15,000 plus an invented age-based add-on structure, leaving a false $6,000 taxable base. Arizona's actual joint deduction for this case eliminates the $28,000 state income base after the Social Security subtraction, so the flat 2.5% rate applies to zero taxable income." -us,scenario_040,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model invented a $200 Arizona senior/property-tax refundable credit based on age and an assumed low Arizona AGI threshold, then applied it without matching the household to any refundable Arizona state-credit computation. PolicyEngine's Arizona refundable-credit variables evaluate to zero for this tax unit, so the model counted a credit pathway that is not available under the 2026 Arizona rules represented in the engine." -us,scenario_040,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable state_refundable_credits value or supporting computation. This is a contract failure rather than a substantive tax calculation, because the required numeric output was missing." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at $16,057 instead of $13,655.95 and treated the $3,753 non-Schedule-D amount as capital-gain income. Those errors inflated AGI above $43,637.65 and produced excessive tax." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model incorrectly used surviving-spouse filing treatment and a $28,900 deduction despite the absence of a qualifying child. It also included Social Security without computing its taxable portion and invented a dependent exemption credit, rather than applying single status and the $24,150 combined deduction." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model excluded all $29,580 of Social Security by treating the dependent-benefit label as income belonging to a nonexistent child. PolicyEngine assigns $13,655.95 of taxable Social Security to the head and also applies the $6,000 senior deduction, yielding $19,487.65 of taxable income." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model used qualifying-surviving-spouse brackets and a roughly $32,350 deduction even though no qualifying child exists. It also overstated taxable Social Security at 85% of benefits instead of $13,655.95." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model assumed the maximum 85% Social Security inclusion, producing AGI near $58,878 instead of $43,637.65. The taxable Social Security amount is $13,655.95, after which the $24,150 combined deduction leaves $19,487.65 of taxable income." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"After correcting its provisional-income calculation, the model still used $16,907 of taxable Social Security instead of $13,655.95 and omitted the $6,000 senior deduction. It therefore taxed about $32,949 rather than the traced $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model alternated between qualifying-surviving-spouse and single treatment, excluded taxable Social Security, and then asserted that deductions eliminated tax despite its own positive taxable-income calculation. Single status, $13,655.95 of taxable Social Security, and $24,150 of deductions produce positive tax of $1,979.16." -us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used $16,906.60 of taxable Social Security and only a $17,425 age-adjusted standard deduction. It omitted the $6,000 senior deduction and consequently calculated $33,287.60 instead of $19,487.65 of taxable income." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied an assumed post-TCJA sunset regime with a $10,300 standard deduction and personal exemption. The applicable computation instead uses an $18,150 standard deduction plus the $6,000 senior deduction, with AGI of $43,637.65." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model applied head-of-household treatment even though the facts support a single filer with no qualifying dependent. Its $3,156 answer also implies that it failed to apply the full $24,150 combination of standard and senior deductions." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used AGI of about $50,581 rather than $43,637.65 and referenced only the elderly single-filer standard deduction. It failed to use the traced taxable Social Security amount and the additional $6,000 senior deduction." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model imposed a pre-TCJA deduction-and-rate regime, allowing only $15,150 of deductions and applying a 15% ordinary bracket. The applicable calculation uses $24,150 of deductions and the 2026 standard rate schedule on $19,487.65 of taxable income." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model incorrectly treated the taxpayer as a qualifying surviving spouse and used a $31,550 deduction. With no qualifying child, single status applies; taxable Social Security is $13,655.95 and total deductions are $24,150." -us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model overstated non-Social-Security income and taxable Social Security, reaching AGI of $52,563, and then omitted the $6,000 senior deduction. The traced inputs produce $43,637.65 of AGI and $19,487.65 of taxable income." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The zero answer incorrectly assumes that age-related deductions or credits eliminate all taxable income. After the $18,150 standard deduction and $6,000 senior deduction, $19,487.65 remains taxable and no nonrefundable credit reduces the resulting tax." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated unspecified deductions and nonrefundable credits as sufficient to reduce liability to zero. The actual allowed deductions total $24,150, leaving $19,487.65 taxable, and no nonrefundable credit is available." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model incorrectly used a roughly $33,200 qualifying-surviving-spouse deduction. The taxpayer files single, and the applicable standard and senior deductions total $24,150, not $33,200." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model overstated AGI at about $58,949 by using an excessive taxable Social Security amount and then taxed qualified income at 15%. AGI is $43,637.65, and the $928 of qualified dividends receives preferential treatment after the $24,150 deduction." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-surviving-spouse deduction and bracket rules despite there being no qualifying child. Single-filer treatment with $24,150 of total deductions leaves $19,487.65 taxable, not roughly $6,036 of ordinary income." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model used a surviving-spouse standard deduction rather than the single-filer $18,150 standard deduction. Combined with the $6,000 senior deduction, the correct deduction is $24,150 and the remaining taxable income extends beyond the 10% bracket." -us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's $3,125 result is consistent with omitting the $6,000 senior deduction or overstating taxable Social Security. The traced calculation uses $43,637.65 of AGI and $24,150 of deductions before applying the rate schedule." -us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied pre-TCJA standard-deduction, personal-exemption, and 15% bracket rules. The applicable 2026 computation uses the $18,150 standard deduction, the $6,000 senior deduction, and the standard schedule on $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at $24,205 and applied a reverted pre-TCJA deduction and 15% bracket. Taxable Social Security is $13,655.95, total deductions are $24,150, and taxable income is $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not be parsed or evaluated substantively." -us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model calculated $24,108.75 of taxable Social Security instead of $13,655.95, inflating AGI to $57,843.75. Although it applied the $6,000 senior deduction, the excessive Social Security inclusion left taxable income far above $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model used married-joint Social Security thresholds and a qualifying-surviving-spouse deduction even though no qualifying child is present. Single status produces $13,655.95 of taxable Social Security and permits $24,150, not $32,150, of deductions." -us,scenario_042,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly assigned qualifying-surviving-spouse status and used the corresponding deduction and brackets. It also included the tax-exempt IRA distribution in provisional income and overstated taxable Social Security; the traced single-filer AGI is $43,637.65." +us,scenario_040,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly identified the elderly/disabled gross-income-test exception but then assumed the mortgage interest and property taxes produced an uncapped shelter deduction large enough for eligibility without completing the net-income calculation. After allowable deductions, the household still fails SNAP's net-income test, and the minimum allotment cannot be awarded to an ineligible household." +us,scenario_040,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model correctly identified that the spouse’s $33,000 bank balance exceeds the stated $2,000 resource limit, then reversed that disqualifying result by invoking home and vehicle exclusions that do not exclude a bank account. Disability and excluded home or vehicle equity do not eliminate the countable-resource test, so the spouse remains outside the aged/disabled pathway." +us,scenario_040,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model tested only the spouse’s $11,200 receipts against a two-person FPL threshold while asserting that individual income and disregards controlled, instead of applying the engine’s pathway-specific income determination, which yields 2.46 times FPL for MAGI categories and no qualifying aged/disabled category. It also described the 69-year-old spouse as “under 65” and treated aged/disabled categorical status as automatic eligibility." +us,scenario_040,spouse_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"The model stated that Arizona’s ABD resource limit is $2,000 for an individual or $3,000 for a couple but nevertheless treated the spouse’s $33,000 bank account as compatible with eligibility. A vehicle exclusion does not shelter cash in a bank account, and disability plus low individual wages does not override the resource and income tests; the spouse qualifies through no Medicaid category." +us,scenario_040,spouse_medicare_eligible,grok-build-0.1,llm_error,age_disability,False,"The model reversed the Medicare age comparison by treating age 69 as under 65. Because the spouse is already older than 65, Medicare eligibility follows directly from age, and the model’s SSDI-duration analysis is irrelevant." +us,scenario_040,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model correctly calculated that the Arizona standard deduction exceeds the $28,000 income remaining after the Social Security subtraction, but then invented a $5,900 residual taxable base despite its own arithmetic. Arizona taxable income is zero, so the pre-refundable-credit tax is zero." +us,scenario_040,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted a modest taxable balance without calculating it. Arizona's Social Security subtraction and married-joint standard deduction reduce taxable income to zero, leaving no amount to which the 2.5% rate applies." +us,scenario_040,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output. +us,scenario_040,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used an approximately $15,000 Arizona joint standard deduction and improvised separate age additions, producing an erroneous $22,000 total deduction. The applicable married-joint standard deduction fully offsets the $28,000 remaining after Arizona excludes Social Security, so the claimed $6,000 taxable base does not exist." +us,scenario_040,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model initially included $21,420 of Social Security in Arizona taxable income instead of subtracting federally taxable Social Security, then combined mutually inconsistent tax, exclusion, and credit calculations to reach an amount larger than its stated intermediate liability. After the Social Security subtraction and joint standard deduction, Arizona taxable income and tax are both zero." +us,scenario_040,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model conflated Arizona’s distinct excise-tax and property-tax credits and treated age plus an asserted $25,000 AGI ceiling as sufficient for a $200 refundable credit. It never applied the specific eligibility requirements for either Arizona credit; this household qualifies for neither, so az_refundable_credits is $0." +us,scenario_040,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted state_refundable_credits entirely instead of submitting the required numeric value and explanation. The required output is $0 because every state and locality refundable-credit component, including Arizona’s, evaluates to zero." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model calculated taxable Social Security as $16,057 instead of $13,655.95 because it included the $3,753 non-Schedule-D capital-gain input in provisional income. That overstated AGI and ordinary taxable income despite correctly applying the $24,150 combined deduction." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model treated the filer as a qualifying surviving spouse and used a married-equivalent deduction even though no qualifying child was listed, so the filing status is single. It also included the full Social Security benefit in income and invented a dependent-exemption credit, instead of taxing only $13,655.95 of Social Security and applying the aged-single and senior deductions." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model wrongly assigned the $29,580 Social Security dependent-benefit input to an unlisted child and excluded it from the head's income. It also omitted the $6,000 senior deduction; the correct calculation includes $13,655.95 of taxable Social Security and total deductions of $24,150." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model used qualifying-surviving-spouse status and married thresholds solely from the surviving-spouse indicator, despite the absence of a qualifying child. The filer is single, and the correct computation applies $24,150 of deductions to $43,637.65 of AGI." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model defaulted taxable Social Security to the 85% maximum, producing $25,143 instead of applying the statutory provisional-income formula, which yields $13,655.95. This inflated AGI by $11,487.05 and consequently overstated the tax." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"After revising its Social Security calculation, the model still included the non-Schedule-D capital-gain and qualified-BDC inputs in the federal income base and omitted the $6,000 senior deduction. The trace instead yields $43,637.65 of AGI and $24,150 of deductions." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model excluded all Social Security by treating the dependent-benefit input as belonging to an unlisted dependent, then asserted that deductions and preferential rates eliminated tax even though its own stated taxable income remained positive. The head has $13,655.95 of taxable Social Security and $19,487.65 of taxable income." +us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model included $16,906.60 of taxable Social Security and applied only a $17,425 aged standard deduction. It missed the $6,000 senior deduction and overstated provisional income; the correct figures are $13,655.95 of taxable Social Security and $24,150 of total deductions." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete post-sunset combination of a reduced standard deduction and personal exemption. For 2026 the computation uses the $18,150 aged-single standard deduction plus the $6,000 senior deduction, with no personal exemption, totaling $24,150." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model says it used a head-of-household deduction even though the facts contain no qualifying person and the filer is single. Its amount also implies that it failed to apply the full $24,150 aged-single standard and senior deductions." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model's approximate $50,581 AGI overstates taxable Social Security and includes income not present in the traced federal gross-income base. It also failed to reduce income by the full $24,150 of aged-single and senior deductions." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly applied a pre-TCJA 15% ordinary bracket and only $15,150 of deductions. The 2026 computation uses the current 10% and 12% schedule on the ordinary portion of $19,487.65 taxable income after $24,150 of deductions." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model used qualifying-surviving-spouse status and a $31,550 married-equivalent deduction despite no qualifying child. The filer is single, taxable Social Security is $13,655.95 rather than $9,846, and total deductions are $24,150." +us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model overstated pre-Social-Security income, treated the non-Schedule-D gain and qualified-BDC input as federal gross income, and consequently calculated $17,757 of taxable Social Security. It then omitted the $6,000 senior deduction, whereas the trace gives $43,637.65 of AGI and $24,150 of deductions." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that age-related deductions and credits fully offset income without computing them. The applicable deductions total $24,150 and leave $19,487.65 of taxable income; no nonrefundable credit eliminates the resulting $1,979.16 tax." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated unspecified deductions or nonrefundable credits as sufficient to reduce tax to zero. The traced deductions are exactly $24,150, no nonrefundable credits apply, and $19,487.65 remains taxable." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model used a roughly $33,200 surviving-spouse deduction, which requires qualifying-surviving-spouse status not supported by any qualifying child in the household. The single filer's correct combined aged standard and senior deductions are $24,150." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $58,949 AGI results from treating 85% of Social Security as taxable and including income outside the traced federal gross-income base. The provisional-income calculation yields $13,655.95 of taxable Social Security and AGI of $43,637.65; the qualified dividends are not taxed at 15% here." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model used qualifying-surviving-spouse deductions and brackets despite the absence of a qualifying child. The filer is single, with $24,150 of deductions and $19,487.65 of taxable income rather than only $6,036 of ordinary taxable income." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model applied a surviving-spouse standard deduction even though the household has no qualifying child and therefore files single. Using the single aged standard deduction plus the $6,000 senior deduction leaves $19,487.65 of taxable income, not an amount confined to the 10% bracket." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's $3,125 answer implies that it did not apply the full $6,000 senior deduction in addition to the aged-single standard deduction. The mortgage balance does not itself create a mortgage-interest deduction, and the correct total deduction is $24,150." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a 2026 TCJA sunset, substituting a reduced standard deduction, a personal exemption, and 10%/15% brackets. The operative calculation uses $24,150 of deductions and the current 10%/12% rate schedule." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at $24,205 and applied a reverted pre-TCJA deduction and 15% bracket. The correct Social Security inclusion is $13,655.95, the deductions total $24,150, and the ordinary income is taxed under the 10%/12% schedule." +us,scenario_042,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model double-counted the same $1,127 farm loss by treating farm income and farm-operations income as separate deductions. It also omitted the $6,000 senior deduction; the trace uses a single $1,127.29 farm-loss deduction and $24,150 of below-AGI deductions." +us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so it failed the required output contract." +us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model computed $24,108.75 of taxable Social Security by misapplying the upper-tier formula to provisional income. The correct taxable Social Security is $13,655.95, producing $43,637.65 of AGI before the correctly identified $24,150 deduction." +us,scenario_042,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model used married Social Security thresholds and a qualifying-surviving-spouse standard deduction even though no qualifying child is present. The filer is single, with $13,655.95 of taxable Social Security and $24,150 of total deductions." +us,scenario_042,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model treated the taxpayer as a qualifying surviving spouse without a qualifying child, used married-equivalent brackets and deductions, and improperly added the tax-exempt IRA distribution to Social Security provisional income. The correct single-filer calculation produces $13,655.95 of taxable Social Security and $24,150 of deductions." +us,scenario_042,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model gave only a generic assertion and a liability exceeding the household's traced taxable income of $19,487.65. The correct rate-schedule calculation on that taxable income, including preferential treatment of $928 of qualified dividends, yields $1,979.16." us,scenario_042,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_042,payroll_tax,gpt-5.4-nano,llm_error,missing_output,False,"The model's explanation applied the correct no-wage rule and stated payroll tax is zero, but it submitted 1530 instead. That violates the output contract because the numeric value contradicts the model's own final explanation." -us,scenario_042,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable payroll_tax value or supporting explanation. The required output was the employee-side payroll tax amount, which is zero because the household has no listed wage or salary income." +us,scenario_042,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no employee wages were listed and that payroll tax should therefore be zero, but submitted $1,530 anyway. That amount contradicts its own payroll-tax-base reasoning and has no listed wage base supporting it." +us,scenario_042,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model provided no payroll-tax output. It failed the required structured-output contract rather than completing the computation from the zero listed wage base. us,scenario_042,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model invented a roughly $24,000 retirement-income exclusion and nevertheless returned tax inconsistent with its own claim that taxable income was near $9,700 and largely covered by the standard deduction. The actual Wisconsin adjustments produce $29,981.71 of Wisconsin AGI and $16,360.34 of taxable income, followed by a $300 property-tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model included all $29,580 of Social Security in Wisconsin income, used an incorrect filing-status deduction, and then treated the property-tax credit as large enough to erase thousands of dollars of tax. Wisconsin taxable income is $16,360.34, generating $584.74 before the fixed $300 nonrefundable credit." -us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model substituted a $5,000 retirement subtraction and extra personal, age, and itemized-deduction credits for the traced Wisconsin calculation. The $700 personal exemption and $250 age exemption reduce taxable income rather than tax, and the applicable nonrefundable property-tax credit is $300, leaving $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $5,000 retirement subtraction, understated the standard deduction, and then invoked unspecified property and itemized-deduction interactions to force the result to zero. The traced deductions yield $16,360.34 of taxable income and only the $300 property-tax credit reduces the $584.74 tax." -us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model started from about $33,735 instead of the traced $43,637.65 federal AGI, applied the wrong capital-gain and standard-deduction treatment, and overstated taxable income at about $27,500. Wisconsin taxable income is $16,360.34, and the $584.74 tax is reduced by $300." -us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model treated a single surviving spouse as married filing jointly, granting a $10,000 retirement subtraction and an approximately $21,151 standard deduction. The applicable traced standard deduction is $12,671.36, while the $950 personal and age exemptions reduce income rather than tax, producing $16,360.34 of taxable income." -us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model used the wrong starting income and standard deduction, then asserted that an itemized-deduction credit based on property and medical expenses could eliminate roughly $790 of tax. The traced taxable income is $16,360.34, and the applicable nonrefundable property-tax credit is exactly $300." -us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model omitted the $950 of personal and age exemptions and stated that no nonrefundable credit applied. Wisconsin applies a $300 property-tax credit to the $584.74 tax computed on $16,360.34 of taxable income." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated Wisconsin taxable income at $29,446 instead of $16,360.34 by failing to reproduce the $13,655.95 of Wisconsin subtractions and $12,671.36 standard deduction. It also mischaracterized the $950 income exemptions as a separate tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $1,137 implies a much larger taxable-income base and no proper application of the $300 property-tax credit. The traced Wisconsin adjustments, standard deduction, and exemptions produce $584.74 before that credit and $284.74 after it." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model mentioned the Social Security subtraction and standard deduction but did not apply the traced amounts or the $950 exemptions and $300 property-tax credit. Those steps reduce the result to $284.74, not $860." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model overstated taxable income at $25,235 by understating the combined Wisconsin subtractions, standard deduction, and $950 exemptions. The correct taxable-income base is $16,360.34, so tax before the $300 property-tax credit is $584.74 rather than $981." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used $33,735 as Wisconsin AGI instead of $29,981.71 and did not subtract the $300 property-tax credit from its stated tax result. Applying the traced $12,671.36 standard deduction and $950 exemptions gives $16,360.34 of taxable income and $284.74 after the credit." -us,scenario_042,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted all IRA and private-pension income merely because the taxpayer was over 65. Wisconsin AGI remains $29,981.71 after the applicable $13,655.95 of subtractions, leaving $16,360.34 taxable after the standard deduction and exemptions." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that pension exclusions and the standard deduction eliminated taxable income. The traced Wisconsin deductions leave $16,360.34 taxable, producing $584.74 of tax and $284.74 after the property-tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The zero answer assumes the retirement-income adjustments, deductions, and credits fully offset liability. They leave $16,360.34 of taxable income, and the sole traced $300 nonrefundable credit only reduces the $584.74 tax to $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model incorrectly concluded that the $300 school-property-tax credit fully offset tentative tax. Tentative Wisconsin tax is $584.74 after the Social Security adjustment, standard deduction, capital-gain treatment, and exemptions, so $284.74 remains." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model failed to identify and subtract the applicable $300 Wisconsin property-tax credit and did not reproduce the traced taxable-income calculation. Tax on the correct $16,360.34 base is $584.74 before that credit and $284.74 afterward." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model invented a $24,000 retirement-income subtraction that eliminated the IRA and pension income. The applicable Wisconsin subtractions total $13,655.95, leaving $29,981.71 of Wisconsin AGI and $16,360.34 of taxable income." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model did not use the traced $13,655.95 Wisconsin subtraction, $12,671.36 standard deduction, and $950 exemptions. Those amounts produce $584.74 of tax before the $300 property-tax credit and $284.74 after it." -us,scenario_042,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the standard deduction and retirement exclusions as sufficient to eliminate all taxable income. The traced calculation leaves $16,360.34 taxable and $284.74 of tax after the property-tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model overstated Wisconsin AGI at $32,680, understated the standard deduction at $10,683, and used a $366 property-tax credit instead of $300. The traced inputs yield $16,360.34 taxable, $584.74 of tax, and $284.74 after the credit." -us,scenario_042,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model omitted the $950 personal and age exemptions and failed to apply the $300 property-tax credit. It also used projected bracket and deduction figures instead of the traced 2026 amounts, which produce $284.74 after credits." -us,scenario_042,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not satisfy the output contract." -us,scenario_042,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the wrong federal AGI, added a 30% capital-gain exclusion to the wrong base, and computed a $12,023 standard deduction instead of $12,671.36. The traced calculation produces $16,360.34 taxable and $584.74 before the $300 property-tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented both a $24,000 retirement-income exclusion and a $24,000 surviving-spouse standard deduction. The traced Wisconsin subtraction is $13,655.95 and the standard deduction is $12,671.36, leaving $16,360.34 taxable." -us,scenario_042,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,The model's arithmetic is internally inconsistent: it described about $297 of tax and credits exceeding that amount but returned $417.88. It also used the wrong filing-status deduction and treated exemptions and an itemized-deduction credit incorrectly; the traced calculation instead yields $584.74 before the $300 property-tax credit. -us,scenario_042,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model invented Wisconsin Homestead Credit eligibility by counting only about $14,420 of income and ignoring the household's full income items, including Social Security benefits, taxable IRA distributions, private pension income, dividends, capital gains, interest, and tax-exempt IRA distributions. Its $1,168 equals 80% of a cited property-tax cap, so it applied a maximum-credit shortcut instead of the actual Wisconsin refundable-credit eligibility and income computation that yields no refundable state credit." -us,scenario_042,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable value or explanation for state_refundable_credits. The required output was a numeric annual amount, and the correct state refundable credit computation yields 0." -us,scenario_043,federal_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model computed the childless EITC phase-in on wages but omitted the childless EITC age ceiling. Because the single filer is 66 and has no qualifying children, the EITC is disallowed before applying the 7.65% earned-income formula." -us,scenario_043,federal_refundable_credits,grok-build-0.1,llm_error,age_disability,False,"The model treated any low-wage worker without children as eligible for the childless EITC and multiplied wages by the 7.65% phase-in rate. It missed that a 66-year-old with no qualifying children is outside the childless EITC age range, so the EITC component is $0." -us,scenario_043,federal_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model explicitly assumed age 66 qualifies for the no-child EITC and then applied the 7.65% phase-in to $4,088 of earned income. The no-child EITC age rule excludes this 66-year-old taxpayer, so the earned-income phase-in calculation never applies." +us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It invented a roughly $24,000 retirement-income exclusion and then reported $953 despite reasoning that the remaining income was nearly absorbed by the standard deduction. The applicable trace instead subtracts $13,655.95 from federal AGI, applies a $12,671.36 standard deduction and $950 of exemptions, and then deducts the $300 property tax credit from $584.74." +us,scenario_042,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It treated the property-related credit as large enough to eliminate several thousand dollars of estimated tax. The applicable nonrefundable Wisconsin property tax credit is $300, reducing $584.74 to $284.74 rather than zero." +us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It incorrectly stacked a $5,000 retirement subtraction, supposed personal-exemption credits, and an itemized-deduction credit until the liability reached zero. Wisconsin instead deducts $950 of exemptions from income and applies the $300 property tax credit to $584.74 of tax, leaving $284.74." +us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used the wrong filing-status deduction, invented a $5,000 retirement subtraction, and treated the $700 and $250 exemptions as vague credits capable of eliminating tax. The traced deductions leave $16,360.34 taxable, and only the $300 property tax credit reduces the resulting $584.74." +us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It understated the sliding-scale standard deduction at about $5,000 and therefore overstated taxable income at roughly $27,500. The applicable standard deduction is $12,671.36, producing $16,360.34 of taxable income before calculating $584.74 of tax and the $300 credit." +us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It treated a single surviving spouse as married filing jointly, assigned a roughly $22,010 standard deduction, and mischaracterized the $950 exemptions as a dollar-for-dollar tax credit. The applicable calculation uses a $12,671.36 standard deduction and subtracts the $950 from income, leaving $16,360.34 taxable." +us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Its own bracket calculation produced about $790, but it then invoked an itemized-deduction credit without computing an allowable amount and used it to erase the liability. The traced nonrefundable reduction is the $300 property tax credit, applied to $584.74 of tax." +us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It omitted the $300 Wisconsin property tax credit despite $5,010.88 of qualifying real estate taxes and explicitly asserted that no nonrefundable credit applied. It also started from $33,806 rather than the traced $29,981.71 Wisconsin AGI." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It left Wisconsin taxable income at approximately $29,446 instead of applying the traced $12,671.36 standard deduction and $950 of exemptions to Wisconsin AGI of $29,981.71. Those deductions produce $16,360.34 taxable and $584.74 of tax before the correctly recognized $300 credit." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its unsupported $1,137 answer implies substantially more taxable income than the traced $16,360.34. It failed to incorporate the full $13,655.95 Wisconsin subtraction, $12,671.36 standard deduction, and $950 of exemptions before applying the $300 credit." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It mentioned the Social Security subtraction and standard deduction but stopped at $860 without applying the complete traced computation. The $16,360.34 taxable amount generates $584.74, and the $300 property tax credit reduces that to $284.74." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It calculated $25,235 of taxable income, overstating the traced amount by $8,874.66 because it did not apply the correct Wisconsin AGI adjustment, standard deduction, and exemptions. The correct taxable amount is $16,360.34 before the $300 property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used $33,735 as Wisconsin AGI rather than $29,981.71 and did not identify the $300 nonrefundable property tax credit. The traced deductions and exemptions yield $584.74 before credits and $284.74 afterward." +us,scenario_042,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It falsely subtracted all IRA and private-pension income merely because the taxpayer is over 65. The traced Wisconsin subtraction is $13,655.95, not the entire $24,086 of retirement income, leaving $16,360.34 taxable after the standard deduction and exemptions." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that pension exclusions and the standard deduction fully offset income. The applicable adjustments leave $16,360.34 taxable, generating $584.74 before the $300 property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its zero assumes the listed retirement income and other deductions fully absorb Wisconsin income. The traced calculation leaves $16,360.34 taxable and $284.74 after the nonrefundable credit." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It correctly identified the property tax credit category but incorrectly concluded that the credit fully offsets tentative tax. The credit is capped at $300, so it reduces $584.74 to $284.74." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It identified no nonrefundable credit and therefore omitted the $300 Wisconsin property tax credit generated by the real estate taxes. It also failed to derive the traced $16,360.34 taxable income and $584.74 pre-credit tax." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It invented a $24,000 retirement-income subtraction that absorbs the IRA and pension income. The applicable Wisconsin subtractions total $13,655.95, leaving $16,360.34 taxable after the standard deduction and exemptions." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"It stopped at an unexplained $430 after deductions and exemptions and did not apply the traced tax-and-credit sequence. Wisconsin tax on $16,360.34 is $584.74, followed by a $300 property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It incorrectly treated retirement exclusions and the standard deduction as sufficient to eliminate taxable income. The applicable adjustments leave $16,360.34 taxable and $284.74 after the property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It started from an overstated $32,680 Wisconsin AGI and used a $10,683 standard deduction, producing $21,047 taxable instead of $16,360.34. It also overstated the school property tax credit at $366; the applicable credit is $300." +us,scenario_042,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,It omitted the $300 property tax credit and used projected rates and deductions rather than the applicable 2026 parameters. The traced computation produces $584.74 before that credit and $284.74 after it. +us,scenario_042,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used an overstated $32,679 Wisconsin income and an understated $11,351 standard deduction, leaving $20,378 taxable. The applicable Wisconsin AGI is $29,981.71 and the standard deduction is $12,671.36, yielding $16,360.34 taxable before the $300 credit." +us,scenario_042,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It supplied no numeric output or explanation for the requested variable. +us,scenario_042,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used a $30% capital-gain exclusion and an independently projected standard-deduction phaseout to reach $19,636 taxable. The traced Wisconsin subtraction and $12,671.36 standard deduction instead leave $16,360.34 taxable; the $300 credit then reduces $584.74 to $284.74." +us,scenario_042,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It invented both a $24,000 retirement-income exclusion and a $24,000 surviving-spouse standard deduction. The applicable subtraction is $13,655.95 and the standard deduction is $12,671.36, leaving $16,360.34 taxable." +us,scenario_042,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its arithmetic is internally inconsistent: it stated approximately $297 of tax and credits of about $32 and $307, yet returned a positive $417.88. It also misclassified the $950 exemptions and itemized deductions; the traced calculation yields $584.74 followed by a $300 property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its unexplained $1,830 implies that it failed to apply the full Wisconsin subtractions, standard deduction, exemptions, and nonrefundable property tax credit. Those steps reduce the requested liability to $284.74." +us,scenario_042,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model understated Wisconsin homestead-credit household income by excluding substantial Social Security, taxable IRA distributions, pension income, dividends, interest, and capital gains, then applied an assumed 2024 maximum-credit formula instead of the 2026 eligibility computation. The complete computation produces no Wisconsin refundable credit." +us,scenario_042,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric state_refundable_credits output or explanation, violating the required structured-output contract." +us,scenario_043,federal_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model treated the age-66 head as eligible for the childless EITC and immediately applied the 7.65% phase-in rate to earned income. Under 2026 law, a taxpayer older than 64 fails the childless EITC age test, so the credit is $0." +us,scenario_043,federal_refundable_credits,grok-build-0.1,llm_error,age_disability,False,"The model calculated 7.65% of $4,088 without first applying the childless EITC age limit. Because the head is 66 and therefore exceeds the maximum eligible age of 64, no EITC is available." +us,scenario_043,federal_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model explicitly asserted that a single filer age 65 or older can receive the 2026 childless EITC, carrying over an inapplicable age rule. The 2026 childless EITC age test excludes this age-66 taxpayer, so applying the 7.65% phase-in rate was erroneous." us,scenario_043,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_043,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_043,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented an income-only aged/disabled pathway for this head. PolicyEngine does not place the head in an aged/disabled category from age and low income alone; age 66 excludes the MAGI adult category, and zero SSI leaves the head with category NONE." -us,scenario_043,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated income below an aged/disabled threshold as sufficient eligibility without establishing the required categorical pathway. The head is outside the expansion-adult age range and receives no SSI, so the engine assigns no Medicaid category." -us,scenario_043,head_medicaid_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model combined the aged/blind/disabled and expansion limits as if either low-income test applied automatically. Age 66 excludes the expansion-adult pathway, and neither disability nor SSI receipt establishes an aged/blind/disabled pathway, yielding category NONE." -us,scenario_043,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly placed a 66-year-old in Colorado’s MAGI adult expansion category, which is age-limited, and then mixed that asset-free pathway with aged/disabled resource limits. The head has no SSI or other qualifying category, so income and the $2,800 asset balance do not produce Medicaid eligibility." -us,scenario_043,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The answer reduces Medicaid eligibility to low income and modest assets without applying a categorical pathway. At age 66 the head is not in the MAGI adult category, and zero SSI plus no other qualifying status produces Medicaid category NONE." -us,scenario_043,head_medicaid_eligible,grok-4.5,llm_error,health_coverage,False,"The model conflated Medicare Savings Program eligibility, such as QMB or SLMB assistance, with the requested PolicyEngine Medicaid eligibility output. Those Medicare cost-sharing pathways do not make `is_medicaid_eligible` true here, and the head’s Medicaid category is NONE." -us,scenario_043,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for `head_medicaid_eligible`. It therefore failed the required output contract rather than completing the Medicaid eligibility computation. +us,scenario_043,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model treated age 65 or older plus income below an aged/disabled limit as sufficient for Medicaid. It never established an aged/disabled eligibility pathway from the supplied facts; the head receives no SSI and the engine assigns category NONE. +us,scenario_043,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model applied an aged/disabled income threshold without first establishing qualification for that Medicaid category. Low income does not supply the missing categorical pathway, and the head receives no SSI, producing category NONE." +us,scenario_043,head_medicaid_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model assumed the head could qualify under either aged/blind/disabled Medicaid or expansion Medicaid based solely on income. At age 66 the head is outside the expansion-adult category, and the facts establish neither blindness/disability nor SSI receipt for another pathway." +us,scenario_043,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly placed a 66-year-old in Colorado's adult expansion category merely because income is below 138% FPL; that category does not cover this age. Its asset discussion cannot create eligibility because the head first fails to enter any Medicaid category, with no SSI or stated disability supporting an aged/disabled pathway." +us,scenario_043,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The answer reduced Medicaid eligibility to very low income and modest assets. Those facts do not establish a qualifying category for this 66-year-old, so the correct categorical result is NONE and eligibility is 0." +us,scenario_043,head_medicaid_eligible,grok-4.5,llm_error,health_coverage,False,"The model conflated eligibility for Medicare Savings Programs such as QMB or SLMB with the requested Medicaid eligibility output. Invoking an MSP resource limit does not establish any Medicaid category in this computation, which assigns the head category NONE." +us,scenario_043,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for head_medicaid_eligible, violating the required output contract." +us,scenario_043,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated household income below a generic Colorado adult Medicaid limit as dispositive. It failed to account for the head's age of 66 and did not identify any aged/disabled or other qualifying pathway, so the engine assigns category NONE." us,scenario_043,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_043,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_043,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_043,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model recognized Colorado FAMLI but used a 0.45% assumption instead of the traced 2026 contribution calculation yielding $17.99, then submitted $315.14 despite its own calculation yielding $331.14. Its final value is unsupported by its stated components and omits most of the required FAMLI contribution." -us,scenario_043,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,The model incorrectly stated that Colorado has no mandatory state payroll tax. It omitted the $17.99 employee FAMLI contribution included in payroll_tax. -us,scenario_043,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"The model identified Colorado FAMLI but wrongly excluded it as an unemployment-insurance-like charge. FAMLI is a mandatory employee state payroll tax in this output, adding $17.99 to federal FICA." -us,scenario_043,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,The model incorrectly asserted that no mandatory Colorado employee payroll tax applies. It omitted the $17.99 FAMLI employee contribution. -us,scenario_043,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,The model stopped after calculating federal Social Security and Medicare taxes. It omitted Colorado’s $17.99 mandatory employee FAMLI contribution. -us,scenario_043,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,The model incorrectly stated that Colorado has no mandatory employee-side state payroll tax. Colorado FAMLI contributes $17.99 to the requested payroll_tax total. -us,scenario_043,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,The model incorrectly treated Colorado as having no applicable mandatory employee payroll contribution. It omitted the $17.99 FAMLI tax. -us,scenario_043,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the entire listed employer-sponsored insurance premium as a pre-tax employee payroll deduction and reduced FICA and FAMLI wages to zero. The traced taxable wage base remains $4,088.15, producing $253.47 of Social Security tax, $59.28 of Medicare tax, and $17.99 of FAMLI tax." -us,scenario_043,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model calculated only the 7.65% federal FICA components. It omitted Colorado’s $17.99 mandatory employee FAMLI contribution. -us,scenario_043,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,The model included Social Security and Medicare but stopped at federal FICA. The requested output also includes the $17.99 Colorado FAMLI employee contribution. -us,scenario_043,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,The model incorrectly stated that Colorado has no applicable state employee payroll tax. It omitted the mandatory $17.99 FAMLI contribution and also rounded the federal components too coarsely. -us,scenario_043,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model treated the reported wages as outside or insufficient for the employee payroll-tax base without identifying any valid exclusion. The traced wages generate $253.47 of Social Security tax, $59.28 of Medicare tax, and $17.99 of Colorado FAMLI tax." -us,scenario_043,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,The model incorrectly excluded mandatory Colorado employee payroll tax and calculated federal FICA alone. Colorado FAMLI adds $17.99. -us,scenario_043,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,The model applied only the combined Social Security and Medicare rate. It omitted the $17.99 Colorado FAMLI employee contribution required in payroll_tax. -us,scenario_043,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model calculated and rounded federal FICA alone. It omitted Colorado’s $17.99 mandatory employee FAMLI contribution. -us,scenario_043,payroll_tax,grok-4.5,llm_error,state_local_rule,False,The model incorrectly asserted that Colorado has no employee payroll tax. The requested output includes a $17.99 Colorado FAMLI contribution in addition to federal FICA. -us,scenario_043,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,The model incorrectly stated that no mandatory state payroll taxes apply. It omitted the $17.99 Colorado FAMLI employee contribution. -us,scenario_043,payroll_tax,minimax-m3,llm_error,state_local_rule,False,The model incorrectly stated that Colorado has no mandatory employee state payroll tax and reported only rounded federal FICA. Colorado FAMLI adds $17.99 to the output. -us,scenario_043,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly stated that Colorado has no mandatory employee state payroll tax. It omitted the $17.99 FAMLI contribution, and its stated rounding from $312.74 to $312.73 was also internally inconsistent." +us,scenario_043,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model recognized that Colorado FAMLI belongs in payroll tax but submitted $315.14, a value unsupported by either its FICA-only calculation or its FICA-plus-FAMLI calculation. It also used an assumed 0.45% employee rate instead of the traced FAMLI computation yielding $17.99." +us,scenario_043,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,The model incorrectly stated that Colorado has no mandatory state payroll tax and therefore omitted the $17.99 employee FAMLI contribution. It also rounded the incomplete federal FICA subtotal to a whole dollar. +us,scenario_043,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model explicitly identified Colorado FAMLI and then excluded it as UI-like, contrary to the requested inclusion of mandatory employee state payroll taxes. That omission reduced the result to federal FICA alone." +us,scenario_043,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,The model incorrectly asserted that no mandatory Colorado employee payroll tax applies. It omitted the $17.99 FAMLI contribution and returned only Social Security and Medicare taxes. +us,scenario_043,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,The model calculated only the 7.65% federal FICA components and omitted Colorado's $17.99 employee FAMLI contribution. Its whole-dollar rounding further obscured the incomplete subtotal. +us,scenario_043,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,The model incorrectly stated that Colorado has no mandatory employee-side state payroll tax. It omitted the $17.99 FAMLI contribution and returned federal FICA alone. +us,scenario_043,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,The model incorrectly treated Colorado as having no applicable mandatory employee payroll contribution. Colorado FAMLI adds $17.99 to the Social Security and Medicare amounts. +us,scenario_043,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the entire listed employer-sponsored insurance premium from FICA and FAMLI wages and forced the taxable wage base to zero. The traced employment wages remain subject to payroll tax, producing all three components totaling $330.73." +us,scenario_043,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,The model used only the 7.65% Social Security and Medicare rate. It omitted Colorado's mandatory employee FAMLI contribution of $17.99. +us,scenario_043,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,The model summed only employee Social Security and Medicare taxes. It failed to add the $17.99 Colorado FAMLI employee contribution required in this payroll-tax output. +us,scenario_043,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,The model incorrectly asserted that Colorado has no applicable employee state payroll tax and omitted the $17.99 FAMLI contribution. It also rounded its incomplete FICA calculation to $312. +us,scenario_043,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model treated the stated wages as outside the employee payroll-tax base without any supporting rule. Those wages generate Social Security, Medicare, and Colorado FAMLI taxes totaling $330.73." +us,scenario_043,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,The model incorrectly excluded all mandatory Colorado employee payroll taxes. The missing Colorado FAMLI contribution is $17.99. +us,scenario_043,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,The model stopped after applying the federal 7.65% Social Security and Medicare rate. It omitted the $17.99 Colorado FAMLI employee contribution. +us,scenario_043,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,The model calculated and rounded only federal Social Security and Medicare taxes. It omitted Colorado's $17.99 mandatory employee FAMLI contribution. +us,scenario_043,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,The model incorrectly concluded that Colorado has no employee payroll tax. Colorado FAMLI contributes $17.99 in addition to federal FICA. +us,scenario_043,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,The model incorrectly stated that no mandatory state payroll tax applies and returned a rounded federal FICA subtotal. It omitted the $17.99 Colorado FAMLI employee contribution. +us,scenario_043,payroll_tax,inkling,llm_error,payroll_tax_base,False,The model considered only employee Social Security and Medicare taxes. It omitted the mandatory Colorado FAMLI employee contribution of $17.99. +us,scenario_043,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,The model incorrectly stated that Colorado has no mandatory employee state payroll tax. It therefore omitted the $17.99 FAMLI contribution and rounded the incomplete FICA subtotal. +us,scenario_043,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,The model incorrectly asserted that Colorado has no mandatory employee state payroll tax and returned only federal FICA. The missing Colorado FAMLI contribution is $17.99. +us,scenario_043,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,The model confused the absence of a general Colorado payroll income tax with the absence of mandatory employee payroll contributions. It omitted the $17.99 Colorado FAMLI contribution. us,scenario_043,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_043,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reduced net income to zero but used an obsolete $291 monthly maximum. PolicyEngine applies the 2026 monthly maxima of $298 to $304.68 and sums them to $3,596.04." -us,scenario_043,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied a $2,250 general resource limit instead of the applicable elderly-household treatment and ignored categorical eligibility through TANF non-cash assistance. The $2,800 account balance is within the allowable limit, so the household receives the maximum allotment." -us,scenario_043,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model derived zero net income and even identified an approximately $300 maximum, then replaced that result with an unsupported $200 monthly estimate. The monthly maximum must be applied directly and summed across the year to $3,596.04." -us,scenario_043,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model recognized that deductions produce a near-maximum benefit but arbitrarily substituted $192 per month after identifying a roughly $292 maximum. Zero net income yields the full monthly maximum, not a reduced allotment." -us,scenario_043,snap,claude-opus-5,llm_error,thresholds_rates,False,The model correctly found net income near zero but used an unsupported $203 monthly allotment. Zero SNAP net income produces the full one-person maximum of $298 to $304.68 per month. -us,scenario_043,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly calculated zero net income but used the obsolete $292 monthly maximum for every month. PolicyEngine uses the applicable 2026 monthly maxima of $298 to $304.68, totaling $3,596.04." -us,scenario_043,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly concluded that the medical deduction eliminates net income but overstated the maximum as $328 per month. The applicable monthly maxima are $298 to $304.68, not $328." -us,scenario_043,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used the FY2025 $292 maximum despite the specified 2026 tax year. The 2026 monthly values range from $298 to $304.68 and sum to $3,596.04." -us,scenario_043,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model correctly reduced net income to zero but treated $292 as the maximum throughout 2025–2026. PolicyEngine applies the 2026 monthly maxima of $298 to $304.68 instead. -us,scenario_043,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"The model incorrectly rejected the household on income and assets without applying the elderly-household rules or categorical eligibility through TANF non-cash assistance. Income is far below the limit and $2,800 in liquid assets is allowable." -us,scenario_043,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly found zero net income but used an obsolete $291 monthly maximum. The applicable 2026 monthly maxima are $298 to $304.68 and total $3,596.04." -us,scenario_043,snap,gemini-3.5-flash,llm_error,period_annualization,False,"The model rounded the annual maximum to $3,500 instead of summing the exact monthly allotments. The 12 monthly values of $298 to $304.68 total $3,596.04." -us,scenario_043,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly calculated zero net income but used a flat $292 monthly maximum. The 2026 monthly maximum changes from $298 to $304.68, producing $3,596.04 annually." -us,scenario_043,snap,glm-5.2,llm_error,thresholds_rates,False,"The model correctly identified entitlement to the maximum allotment but used a flat $302 monthly value, as implied by $3,624 divided by 12. PolicyEngine instead applies monthly maxima ranging from $298 to $304.68." -us,scenario_043,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model acknowledged eligibility but supplied an unsupported annual estimate equivalent to only about $106.33 per month. The deductions reduce net income to zero, requiring the full monthly maxima and an annual total of $3,596.04." -us,scenario_043,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model treated the absence of unspecified low-income indicators as disqualifying even though the listed income is far below the SNAP limit. It also omitted the elderly member's categorical eligibility through TANF non-cash assistance and the deductions that reduce net income to zero. -us,scenario_043,snap,gpt-5.5,llm_error,period_annualization,False,"The model correctly found zero net income but multiplied the initial $298 monthly maximum by all 12 months. PolicyEngine applies later monthly values up to $304.68, so the exact annual sum is $3,596.04." -us,scenario_043,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The model froze the maximum allotment at $298 for the entire year. The monthly parameter rises as high as $304.68, and summing the actual 12 monthly amounts yields $3,596.04." -us,scenario_043,snap,gpt-5.6-sol,llm_error,period_annualization,False,The model applied $298 uniformly across all 12 months after correctly finding zero net income. It missed the within-year increase in the monthly maximum to as much as $304.68. -us,scenario_043,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model correctly derived zero net income but annualized only the initial $298 monthly maximum. PolicyEngine sums varying monthly maxima of $298 to $304.68 to reach $3,596.04." -us,scenario_043,snap,grok-4.3,llm_error,asset_resource,False,"The model incorrectly treated $2,800 in assets as exceeding the applicable limit. The elderly household is within the allowable resource limit and is categorically eligible through TANF non-cash assistance." -us,scenario_043,snap,grok-4.5,llm_error,thresholds_rates,False,The model correctly established zero net income and full-benefit eligibility but used an obsolete $292 monthly maximum. The applicable 2026 values range from $298 to $304.68. -us,scenario_043,snap,grok-build-0.1,llm_error,period_annualization,False,"The model rounded the maximum to $300 per month and multiplied by 12. The exact monthly values vary from $298 to $304.68 and sum to $3,596.04, not $3,600." -us,scenario_043,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP output or explanation, violating the required output contract." -us,scenario_043,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly found zero net income but applied $298 in every month. It omitted the within-year increase to as much as $304.68, which raises the annual total to $3,596.04." -us,scenario_043,snap,minimax-m3,llm_error,asset_resource,False,"The model asserted a zero result despite income far below the limit and $2,800 of assets within the elderly-household allowance. It failed to apply categorical eligibility through TANF non-cash assistance and the deductions that produce the maximum allotment." -us,scenario_043,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used an unsupported $243 monthly maximum after deriving very low net income and also mislabeled medical costs as an excess shelter deduction. The elderly medical deduction reduces net income to zero, so the full monthly maxima of $298 to $304.68 apply." -us,scenario_043,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"The model treated categorical eligibility at age 66 as sufficient and mechanically subtracted its own annualized SSI exclusions from wages and pension income to produce a residual benefit. It failed to apply the SSI income-limit test under which this individual’s approximately $4,642 of income eliminates entitlement." -us,scenario_043,ssi,claude-opus-5,llm_error,thresholds_rates,False,"The model converted wages and pension income into approximately $2,541 of annual countable income and subtracted that from an assumed annual federal benefit rate, even though the applicable SSI income-limit test yields no eligibility. Its unexplained reduction from its own $9,360 calculation to $6,720 also has no valid SSI computation step." -us,scenario_043,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract." -us,scenario_043,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model stopped after denying the federal-linked Colorado EITC and child credits. It omitted Colorado's sales tax refund, for which the age-66 filer and $4,072.71 of employment income qualify, yielding $19 at $4,625.40 of modified adjusted gross income." -us,scenario_043,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that Colorado has no applicable refundable individual credits. It omitted the refundable Colorado sales tax refund and therefore missed the $19 amount determined from the filer's senior status, employment income, and modified adjusted gross income." -us,scenario_043,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model substituted an estimated $1,130 TABOR refund for the applicable Colorado sales tax refund. The relevant program uses the qualifying senior filer's $4,625.40 modified adjusted gross income to produce $19, not a projected TABOR tier amount." -us,scenario_043,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model treated low wages and the absence of children as disqualifying for all Colorado refundable credits. It omitted the sales tax refund pathway for a filer age 66 with employment income, which produces $19." -us,scenario_043,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model assigned an unsupported $225 senior housing or income-qualified credit instead of calculating the Colorado sales tax refund. The applicable senior-and-employment-income pathway yields $19 from modified adjusted gross income of $4,625.40." -us,scenario_043,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model correctly eliminated the Colorado EITC and child credit but incorrectly concluded that no other refundable credit applied. It omitted the Colorado sales tax refund available to this age-66 filer with employment income, worth $19." -us,scenario_043,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model conflated the Colorado sales tax refund with TABOR and senior housing provisions and estimated a combined $1,200 without applying the sales-tax-refund schedule. The qualifying senior's $4,625.40 modified adjusted gross income yields a $19 sales tax refund." -us,scenario_043,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model treated the absence of a federal EITC and qualifying children as eliminating every Colorado refundable credit. It omitted the separate Colorado sales tax refund, which pays this qualifying senior filer $19." -us,scenario_043,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model considered only the Colorado EITC and child-related credits. It failed to apply the Colorado sales tax refund for the age-66 filer with employment income, which yields $19." -us,scenario_043,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model's blanket denial of eligibility omitted Colorado's sales tax refund pathway. The filer's age and employment income establish eligibility, and $4,625.40 of modified adjusted gross income produces a $19 refund." -us,scenario_043,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model treated the absence of qualifying property-tax, rent, or heat expenses for the PTC rebate as dispositive. Colorado's separate sales tax refund does not depend on those reported expenses, and this qualifying senior receives $19." -us,scenario_043,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,The model incorrectly made federal EITC eligibility a prerequisite for all Colorado refundable credits. The Colorado sales tax refund has a separate senior-and-employment-income pathway that yields $19 here. -us,scenario_043,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The zero answer omitted the Colorado sales tax refund. The age-66 filer qualifies with employment income, and the applicable modified-adjusted-gross-income amount produces a $19 refundable credit." -us,scenario_043,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model stopped after correctly denying the childless federal and Colorado EITC because of age. It omitted the independent Colorado sales tax refund, under which the same age-66 status and employment income produce $19." -us,scenario_043,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model's claim that no reported fact triggered a refundable Colorado credit omitted the sales tax refund. Age 66 and employment income trigger that program, with $4,625.40 of modified adjusted gross income yielding $19." -us,scenario_043,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,The model restricted refundable credits to earned-income or child-based programs and overlooked Colorado's sales tax refund for qualifying seniors with employment income. The resulting refund is $19. -us,scenario_043,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model searched for child, EITC, and property/rent/heat credits but omitted Colorado's distinct sales tax refund. The age-66 filer qualifies through employment income and receives $19 based on modified adjusted gross income." -us,scenario_043,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,The zero answer omitted the Colorado sales tax refund eligibility pathway. The filer's senior status and employment income qualify the household for a $19 refund. -us,scenario_043,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model incorrectly treated federal EITC, dependents, and listed expenses as the only possible bases for a Colorado refundable credit. It omitted the sales tax refund for an age-66 filer with employment income, which equals $19." -us,scenario_043,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,The model failed to recognize that the listed age and employment income support Colorado sales tax refund eligibility. Applying the program's modified-adjusted-gross-income schedule yields $19. -us,scenario_043,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The blanket zero omitted Colorado's sales tax refund. The age-66 filer with employment income qualifies, and the refund schedule produces $19." -us,scenario_043,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model stopped after excluding the Colorado EITC and child-related credits. It omitted the separate Colorado sales tax refund for qualifying senior filers, worth $19 here." -us,scenario_043,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model incorrectly awarded a Colorado EITC based on $313 of federal EITC even though the childless filer is age 66 and fails the federal EITC age rule. It also omitted the applicable Colorado sales tax refund, whose correct computation yields $19." -us,scenario_043,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model treated the lack of dependents and qualifying expenses as excluding every Colorado refundable credit. It omitted the sales tax refund pathway based on senior status, employment income, and modified adjusted gross income, which yields $19." -us,scenario_043,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,The model limited its analysis to federal-linked and child or child-care credits. Colorado's separate sales tax refund applies to this age-66 filer with employment income and equals $19. -us,scenario_043,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model overlooked age 66 and employment income as qualifying characteristics for Colorado's sales tax refund. Applying the refund schedule to $4,625.40 of modified adjusted gross income produces $19." -us,scenario_043,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model focused on the Colorado child credit and EITC and incorrectly concluded that no other qualifying characteristic existed. The filer's age 66 and employment income qualify the household for a $19 Colorado sales tax refund. +us,scenario_043,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reduced net income to zero but used an outdated $291 monthly maximum. PolicyEngine applies the 2026 monthly maximums of $298–$304.68 and sums them to $3,596.04." +us,scenario_043,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied a $2,250 resource limit and denied eligibility because $2,800 exceeded it. This elderly household is categorically eligible through TANF non-cash assistance, and its $2,800 of liquid assets is within the applicable allowable limit." +us,scenario_043,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model derived zero net income and even identified a maximum near $300 per month, then replaced that result with an unsupported $200 monthly estimate. The full monthly maximum schedule produces $3,596.04 annually." +us,scenario_043,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model established near-zero net income but arbitrarily substituted $192 per month for the approximately $292 maximum it had identified. Zero net income entitles the household to the full 2026 monthly allotments, totaling $3,596.04." +us,scenario_043,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly concluded that deductions reduce net income to zero but used $203 per month as the maximum. PolicyEngine's one-person maximum varies from $298 to $304.68 during 2026 and totals $3,596.04." +us,scenario_043,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly found zero net income and maximum-benefit eligibility but froze the allotment at $292 per month. It missed the applicable 2026 monthly maximums of $298–$304.68, whose annual sum is $3,596.04." +us,scenario_043,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly used the elderly medical deduction to reach zero net income but overstated the maximum as $328 per month. The applicable monthly maximums are $298–$304.68, totaling $3,596.04." +us,scenario_043,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used the FY2025 maximum of $292 per month despite the 2026 tax year. PolicyEngine applies the 2026 monthly schedule and returns $3,596.04." +us,scenario_043,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly reduced net income to zero but treated $292 as the maximum throughout 2025–2026. The 2026 calculation uses monthly maxima of $298–$304.68, totaling $3,596.04." +us,scenario_043,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model denied SNAP despite gross income of only about $387 per month, allowable $2,800 liquid assets, and categorical eligibility through TANF non-cash assistance for an elderly household member. Those rules produce eligibility for the full maximum allotment." +us,scenario_043,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly found zero net income but used a $291 monthly maximum. PolicyEngine's applicable 2026 maximums vary from $298 to $304.68 and sum to $3,596.04." +us,scenario_043,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly identified categorical eligibility and zero net income but rounded the annual maximum to $3,500 instead of applying the monthly schedule. Summing the exact 2026 allotments yields $3,596.04." +us,scenario_043,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly found zero net income but applied the obsolete $292 monthly maximum for every month. The applicable monthly amounts are $298–$304.68 and total $3,596.04." +us,scenario_043,snap,glm-5.2,llm_error,thresholds_rates,False,"The model correctly determined that the household receives the maximum but implicitly used a flat $302 per month. PolicyEngine applies varying monthly maxima of $298–$304.68, producing $3,596.04 rather than $3,624." +us,scenario_043,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model recognized eligibility but substituted an unsupported annual estimate of $1,276 without computing net income or the maximum allotment. The deductions reduce net income to zero, and the full monthly maximums sum to $3,596.04." +us,scenario_043,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of unspecified low-income indicators as disqualifying even though the listed income, age, assets, and categorical-eligibility pathway establish SNAP eligibility. The household receives the maximum allotment because its SNAP net income is zero." +us,scenario_043,snap,gpt-5.5,llm_error,period_annualization,False,"The model correctly found zero net income but annualized $298 for all 12 months. PolicyEngine uses $298 in some months and up to $304.68 in others, so the annual total is $3,596.04." +us,scenario_043,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The model correctly established resource eligibility and zero countable income but held the $298 maximum constant for the entire year. Summing PolicyEngine's changing monthly maxima gives $3,596.04." +us,scenario_043,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model correctly concluded that the maximum applies but multiplied a single $298 monthly amount by 12. The monthly maximum rises as high as $304.68 during the year, making the annual benefit $3,596.04." +us,scenario_043,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model correctly reduced net income to zero but annualized the initial $298 maximum without applying the within-year schedule. The exact monthly amounts total $3,596.04." +us,scenario_043,snap,grok-4.3,llm_error,asset_resource,False,"The model denied SNAP because it treated $2,800 as exceeding the applicable resource limit. The elderly household's assets are allowable, and categorical eligibility through TANF non-cash assistance independently satisfies the relevant pathway." +us,scenario_043,snap,grok-4.5,llm_error,thresholds_rates,False,"The model correctly found eligibility and zero net income but used $292 per month. PolicyEngine's 2026 maximums are $298–$304.68 across the year and sum to $3,596.04." +us,scenario_043,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model rounded the one-person maximum to $300 per month instead of using the exact monthly parameters. The applicable amounts vary from $298 to $304.68 and total $3,596.04." +us,scenario_043,snap,inkling,llm_error,thresholds_rates,False,"The model correctly found zero net income but replaced the exact maximum-allotment schedule with an approximate $300 monthly amount. The exact annual sum is $3,596.04." +us,scenario_043,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP output, so it failed the required output contract." +us,scenario_043,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly found zero net income but multiplied $298 by all 12 months. PolicyEngine's maximum increases to as much as $304.68 during the year, producing $3,596.04." +us,scenario_043,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model acknowledged the elderly eligibility rules but asserted a zero benefit without applying them. Gross income is far below the limit, $2,800 of assets is allowable, and TANF non-cash categorical eligibility leads to the full maximum allotment." +us,scenario_043,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model called employer insurance premiums an excess shelter deduction and then failed to subtract the listed elderly medical expenses fully, leaving positive net income despite deductions exceeding income. It also used an understated $243 monthly maximum; zero net income and the exact monthly maximums produce $3,596.04." +us,scenario_043,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted that PolicyEngine produces no benefit without applying the stated income, resource, deduction, and categorical-eligibility rules. Those rules establish eligibility and zero net income, yielding the full $3,596.04 allotment." +us,scenario_043,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"The model skipped the SSI income-limit eligibility gate and instead reduced the federal benefit rate by its own countable-income calculation. PolicyEngine applies the individual income test first; the person’s approximately $4,642 of income exceeds the limit, so the payable SSI amount is zero." +us,scenario_043,ssi,claude-opus-5,llm_error,thresholds_rates,False,"The model treated SSI as a benefit-rate-minus-countable-income calculation despite the person failing the financial eligibility threshold. It then submitted $6,720 without deriving that figure from its own stated benefit rate and countable income, which produced approximately $9,360 rather than $6,720." +us,scenario_043,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI output or explanation, so it failed the required submission contract." +us,scenario_043,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model treated the Colorado EITC and child-related credits as the complete set of refundable credits. It omitted the Colorado sales tax refund, for which the age-66 filer’s employment income and $4,625.40 modified AGI produce $19." +us,scenario_043,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,The model incorrectly asserted that Colorado has no refundable individual credits. It omitted the refundable Colorado sales tax refund and therefore failed to apply the $19 amount based on this senior filer’s employment income and modified AGI. +us,scenario_043,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model substituted an estimated TABOR refund for the Colorado sales tax refund and applied an unsupported $1,130 amount. The applicable program uses the filer’s qualifying status and $4,625.40 modified AGI to yield $19." +us,scenario_043,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model incorrectly concluded that low wages and the absence of children eliminated every Colorado refundable credit. It omitted the Colorado sales tax refund available to this age-66 filer, which equals $19 at the household’s modified AGI." +us,scenario_043,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model invented a $225 senior housing or income-qualified credit without applying its actual eligibility inputs. The qualifying refundable item is the Colorado sales tax refund, whose income schedule produces $19." +us,scenario_043,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model correctly excluded the childless EITC pathway at age 66 but incorrectly stopped after checking EITC and child credits. It omitted the separate Colorado sales tax refund, which yields $19 from the filer’s employment income and modified AGI." +us,scenario_043,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model combined speculative TABOR and senior-housing credits into an unsupported $1,200 estimate. It failed to calculate the distinct Colorado sales tax refund, which is $19 for this filer under the modified-AGI schedule." +us,scenario_043,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model treated the absence of an EITC-qualifying child as eliminating all Colorado refundable credits. It omitted the Colorado sales tax refund, whose senior and income rules produce $19." +us,scenario_043,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model checked only the Colorado EITC and child-related credits. It omitted the independent Colorado sales tax refund, which provides $19 based on the filer’s age, employment income, and modified AGI." +us,scenario_043,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model incorrectly concluded that the household’s income and composition satisfy no Colorado refundable-credit pathway. The age-66 filer qualifies for the Colorado sales tax refund, and $4,625.40 of modified AGI yields $19." +us,scenario_043,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model evaluated only the property-tax/rent/heat rebate and treated the absence of its expenses as dispositive. It omitted the separate Colorado sales tax refund, which requires no reported housing expense and equals $19 here." +us,scenario_043,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,The model incorrectly made federal EITC eligibility a prerequisite for every Colorado refundable credit. The Colorado sales tax refund is a separate pathway and produces $19 for this filer. +us,scenario_043,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The unexplained zero omits the Colorado sales tax refund. Applying its senior, employment-income, and modified-AGI rules yields $19." +us,scenario_043,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model correctly ruled out the childless federal and Colorado EITC at age 66 but incorrectly concluded that no other refundable credit applies. It omitted the Colorado sales tax refund, which equals $19." +us,scenario_043,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model’s zero reflects a failure to test the Colorado sales tax refund pathway. The age-66 filer’s employment income establishes eligibility, and the modified-AGI schedule yields $19." +us,scenario_043,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model limited refundable credits to programs requiring children or an earned-income-credit calculation. It omitted the Colorado sales tax refund for qualifying seniors, which produces $19 here." +us,scenario_043,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model checked child, EITC, and property/rent/heat pathways but omitted Colorado’s sales tax refund. That program applies to this age-66 filer and awards $19 at the household’s modified AGI." +us,scenario_043,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model’s blanket conclusion omitted the Colorado sales tax refund. Applying the filer’s age, employment income, and $4,625.40 modified AGI produces $19." +us,scenario_043,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model incorrectly treated federal EITC, dependents, and listed expenses as the only routes to a Colorado refundable credit. The separate Colorado sales tax refund requires none of those and yields $19 here." +us,scenario_043,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,The model failed to recognize that age 66 and employment income trigger the Colorado sales tax refund pathway. The applicable modified-AGI tier produces a $19 refundable credit. +us,scenario_043,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The unsupported zero omits the Colorado sales tax refund. The filer qualifies through the program’s senior and income rules, producing $19." +us,scenario_043,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model treated the lack of federal EITC and children as eliminating all Colorado refundable credits. It omitted the independent Colorado sales tax refund, which equals $19 for this household." +us,scenario_043,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model incorrectly awarded a Colorado EITC by applying 50% to an erroneous $313 federal EITC despite the age-66 childless filer’s ineligibility. It also omitted the actual Colorado sales tax refund, which is $19." +us,scenario_043,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model correctly found no child credit or age-qualified childless EITC but incorrectly stopped there. It omitted the separate Colorado sales tax refund, which produces $19." +us,scenario_043,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,The model incorrectly treated the lack of dependents and qualifying expenses as eliminating all refundable credits. The Colorado sales tax refund is available through the filer’s age and income characteristics and equals $19. +us,scenario_043,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model limited its analysis to credits tied to federal EITC, CTC, children, or childcare expenses. It omitted the Colorado sales tax refund, which independently yields $19." +us,scenario_043,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model incorrectly concluded that low income and no children provide no qualifying factor. The filer’s age and employment income qualify the household for Colorado’s $19 sales tax refund. +us,scenario_043,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model checked the Colorado Child Tax Credit and EITC but overlooked the age-based Colorado sales tax refund pathway. Applying its modified-AGI schedule yields $19. +us,scenario_043,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model incorrectly used zero Colorado income-tax liability as a reason to deny refundable credits. Refundability allows the Colorado sales tax refund to be paid despite zero liability, and the household receives $19." us,scenario_043,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_044,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the full $56,806 of Social Security retirement income as taxable income before the standard deduction. It skipped the married-filing-jointly Social Security provisional-income calculation, under which no Social Security benefits are taxable here, so taxable income is $0 and federal income tax before refundable credits is $0." -us,scenario_044,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model computed a positive income tax liability from Social Security benefits after applying an older-adult standard deduction, which means it included taxable Social Security that the provisional-income test excludes. With no other income, the joint filers' provisional income is below the taxable Social Security threshold, leaving no taxable income and no nonrefundable-credit offset to apply." -us,scenario_044,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed 85% of the $56,806 Social Security benefit was taxable instead of applying the joint-filer provisional-income formula. It also submitted $5,816.40 despite its own bracket calculation producing $1,808.51, but the decisive substantive error is that taxable Social Security is $0, so the regular income tax before credits is $0." -us,scenario_044,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model taxed Social Security retirement income as though it were employee wage income subject to Social Security and Medicare payroll taxes. It ignored the prompt rule that unlisted wage income is zero and the household facts list only Social Security retirement income, leaving no employee payroll-tax base." -us,scenario_044,spouse_medicare_eligible,gpt-5.4-nano,llm_error,other,False,"The model's explanation applied the age rule correctly: a 61-year-old spouse without any listed qualifying condition is not Medicare eligible. Its submitted numeric value contradicted that explanation by returning 1, so the failure is an answer-contract mismatch rather than a substantive Medicare rule error." -us,scenario_044,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model applied Kansas income tax brackets to Social Security retirement income after generic exemptions instead of subtracting the Social Security benefits from Kansas taxable income. With no other listed taxable income, the Kansas tax base is $0 and no Kansas bracket tax is due." -us,scenario_044,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated some portion of the reported Social Security as taxable under a standard-deduction calculation. Kansas excludes the household's Social Security benefits from the state taxable base in 2026, leaving no taxable Kansas income and no pre-refundable-credit state income tax." -us,scenario_044,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model explicitly taxed Social Security benefits as ordinary Kansas income, then applied Kansas deductions, exemptions, and rate brackets to $44,306 of invented taxable income. The correct computation removes the $56,806 of Social Security retirement income from Kansas taxable income, so the bracket schedule never applies." -us,scenario_044,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model set Kansas AGI to $0 merely because the household's only listed income is Social Security retirement income, then awarded two $125 food sales tax credits. PolicyEngine's Kansas income computation does not place this $56,806-income household below the credit's applicable income limit, so the food sales tax credit is $0." -us,scenario_044,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model asserted that Kansas income was below the food sales tax credit limit without correctly deriving Kansas income from the household's $56,806 of Social Security retirement income. That erroneous income treatment triggered two $125 exemption amounts, while the correct Kansas refundable-credit computation yields $0." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived taxable income of about $20,176 and tax of about $2,173 using the applicable standard deduction and brackets, then submitted $2,540 instead. Its final value contradicts its own completed computation." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $21,208 employer-sponsored insurance premium from the supplied gross wages, reducing AGI to $15,068. It then incorrectly used refundable EITC to reduce an output expressly measured before refundable credits; the correct taxable income is $20,175.59 and the resulting tax is $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model rechecked the calculation and correctly obtained approximately $2,173 from $20,176 of taxable income, but submitted $1,631. The submitted number has no basis in its stated derivation." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly calculated $1,240 plus $933, or approximately $2,173, and correctly found no applicable nonrefundable credits. It then submitted $1,387, contradicting its own arithmetic." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived approximately $2,173 using the $16,100 standard deduction and the 10% and 12% brackets, then replaced that result with $2,029 under the label of rounding. Rounding cannot account for a $144 discrepancy." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an obsolete estimated $15,350 standard deduction and an $11,925 first-bracket cutoff instead of the 2026 values applied in the calculation. The $16,100 deduction leaves $20,175.59 taxable income, whose main-schedule tax is $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's own estimated inputs produced about $2,226 of tax, but it submitted $1,130 without applying any identified credit or tax rule. The correct deduction and rate schedule produce $2,173.07, and there is no nonrefundable credit supporting the submitted reduction." -us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the $21,208 ESI premium from gross wages and incorrectly applied a post-TCJA-sunset standard deduction plus personal exemption. The applicable 2026 calculation uses $36,275.59 of AGI and a $16,100 standard deduction, with no personal exemption, leaving $20,175.59 taxable." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model double-counted the ESI premium as a deduction from the supplied gross wages and therefore placed income below the standard deduction. AGI remains $36,275.59; after the $16,100 standard deduction, $20,175.59 is taxable and generates $2,173.07 of tax." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model called the taxpayer a head-of-household filer even though no qualifying person is listed and the filing status is single. It also failed to tax the income remaining after the deduction: the single-filer deduction leaves $20,175.59 taxable and $2,173.07 due." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a pre-TCJA regime with a personal exemption and 15% bracket instead of the applicable 2026 $16,100 standard deduction and 10%/12% schedule. Those rules leave $20,175.59 taxable and produce $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by the full ESI premium and then used a standard-deduction-plus-personal-exemption regime. The calculation instead starts from $36,275.59 of AGI, deducts $16,100 with no personal exemption, and taxes $20,175.59." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model double-deducted the employer-sponsored insurance premium and incorrectly assumed the TCJA individual provisions had expired, introducing a personal exemption and smaller standard deduction. The applicable computation uses $36,275.59 of AGI, a $16,100 standard deduction, and no personal exemption." -us,scenario_045,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly kept gross wages in AGI but used estimated deduction and bracket values rather than the applicable $16,100 standard deduction and 2026 rate thresholds. Applying the actual schedule to $20,175.59 of taxable income yields $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that the standard deduction and unspecified nonrefundable credits eliminate the tax, but the facts supply no applicable nonrefundable credit. The $16,100 standard deduction leaves $20,175.59 taxable, producing $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed unspecified nonrefundable credits fully offset the liability even though no qualifying credit facts are present. After the $16,100 standard deduction, $20,175.59 remains taxable and generates $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an estimated $15,750 standard deduction instead of $16,100 and did not carry its stated bracket arithmetic consistently into the submitted $2,215. The applicable deduction and schedule produce $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $21,208 of employer-sponsored insurance premiums from the supplied gross wages, reducing taxable wages to $15,068. AGI remains $36,275.59, so the $16,100 standard deduction leaves $20,175.59 taxable rather than zero." -us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an approximate $15,000 standard deduction rather than the applicable $16,100 amount and misstated the resulting bracket arithmetic. Taxing $20,175.59 under the 2026 schedule produces $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model double-deducted the ESI premium and incorrectly applied a post-TCJA-sunset standard deduction and personal exemption. The applicable rules retain $36,275.59 as AGI, allow a $16,100 standard deduction, and provide no personal exemption, resulting in $2,173.07 of tax." -us,scenario_045,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly assumed a post-TCJA-sunset regime, using a small standard deduction, a personal exemption, and a 15% second bracket. The applicable 2026 rules use the $16,100 standard deduction, no personal exemption, and the main 10%/12% schedule, yielding $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not satisfy the output contract." -us,scenario_045,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model correctly found about $20,176 of taxable income but then set the liability to zero based on EITC and unspecified credits. EITC is refundable and is excluded from this before-refundable-credits output, while no applicable nonrefundable credit exists; the tax remains $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's stated bracket calculation produced $2,220.12 with no nonrefundable credits, but it submitted $424.76. The submitted amount is disconnected from its derivation; using the applicable $16,100 deduction and rate schedule gives $2,173.07." -us,scenario_045,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly recognized the childless EITC pathway and that its phaseout ends near $20,000, but then awarded $632 at $36,276 despite the credit already being fully phased out. Applying the childless EITC phaseout to the stated wages yields $0." -us,scenario_045,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $21,208 employer-sponsored insurance premium from the stated $36,276 of gross wages to manufacture lower AGI and an EITC. The prompt supplies gross annual wages and no employee pre-tax premium deduction, so earned income remains $36,276 and the childless EITC is $0." -us,scenario_045,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model treated $36,276 minus the $21,208 employer-sponsored insurance premium as $15,068 of earned income. Those premiums are not an employee wage deduction under the supplied facts, so the EITC calculation must use $36,276, where the childless credit is fully phased out." -us,scenario_045,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model invented dependent status even though the household contains only one adult and all unlisted household facts are false. With no qualifying child and $36,276 of wages, the filer receives neither a child-based EITC nor a childless EITC, so refundable credits are $0." -us,scenario_045,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used $15,068 as earned income, which is the stated $36,276 of gross wages minus the $21,208 employer-sponsored insurance premium. The premium is not a supplied employee pre-tax wage deduction, so earned income is $36,276 and the childless EITC is fully phased out." -us,scenario_045,federal_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model reduced both AGI and earned income to $15,068 by subtracting the employer-sponsored insurance premium from gross wages. The EITC must use the full $36,276 of wages under these facts, which places the filer above the childless EITC phaseout endpoint." -us,scenario_045,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model explicitly subtracted the $21,208 employer insurance premium from $36,276 of gross wages and treated the remainder as earned income and AGI. No employee pre-tax premium deduction is listed, so the childless EITC is computed from $36,276 and equals $0." -us,scenario_045,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model calculated the childless EITC using $15,068, obtained by subtracting the listed employer-sponsored insurance premium from gross wages. The prompt does not specify that premium as an employee pre-tax payroll deduction, so earned income remains $36,276 and the credit is fully phased out." -us,scenario_045,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, violating the required submission contract." +us,scenario_044,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the standard deduction directly from the full $56,806 Social Security benefit, effectively treating gross Social Security as taxable income. It failed to calculate taxable Social Security from provisional income before applying the joint-filer deduction, which leaves no taxable income." +us,scenario_044,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,age_disability,False,"The model incorrectly described both spouses as older adults even though the spouse is 61, then produced tax without deriving the taxable portion of Social Security or taxable income. Only the 66-year-old head receives the age-based treatment, and the household's taxable Social Security remains below the applicable joint-filer deductions, producing zero tax." +us,scenario_044,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated 85% of the entire Social Security benefit as taxable, ignoring that provisional income with no other income equals only half the benefits and is only modestly above the $25,000 joint base amount. It also computed $1,808.51 in its explanation but submitted $5,816.40, a final value unsupported by its own arithmetic." +us,scenario_044,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model correctly stated that its claimed $24,730 taxable Social Security was below its $34,250 deduction and therefore produced zero taxable income, but then separately taxed that Social Security despite the deduction. It further misclassified the senior deduction as a nonrefundable credit and concluded with $0 while submitting $4,234, contradicting its own derivation." +us,scenario_044,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that the household has no wage or self-employment income and therefore no employee payroll-tax base, but then contradicted that reasoning by imposing $2,926 of unspecified “typical” wage-related payroll taxes. The $56,806 is Social Security retirement income, which is not subject to employee Social Security or Medicare tax, so the correct payroll-tax computation yields $0." +us,scenario_044,spouse_medicare_eligible,gpt-5.4-nano,llm_error,other,False,"The model correctly stated that the 61-year-old spouse is not Medicare eligible and explicitly derived 0, then submitted value = 1. It violated the required consistency between the eligibility explanation and numeric output." +us,scenario_044,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated Social Security retirement benefits as Kansas-taxable income and applied state brackets after exemptions. Kansas subtracts the federally taxable Social Security amount from Kansas adjusted gross income, leaving no taxable income and $0 tax before refundable credits." +us,scenario_044,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model subjected the reported Social Security income to Kansas tax after a standard-deduction-style adjustment. It omitted Kansas’s subtraction for federally taxable Social Security benefits, which removes the household’s only income from the Kansas tax base and yields $0." +us,scenario_044,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model explicitly asserted that Kansas taxes Social Security as ordinary income, then reduced the full $56,806 only by an $8,000 standard deduction and $4,500 of personal exemptions before applying tax brackets. Kansas instead subtracts the federally taxable Social Security benefits from Kansas adjusted gross income, so none of that benefit reaches the bracket calculation and the resulting tax is $0." +us,scenario_044,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model classified the Kansas food sales tax credit as refundable and added $125 for each of two exemptions to state_refundable_credits. That credit is nonrefundable, so its asserted eligibility does not create any amount in this output." +us,scenario_044,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model incorrectly placed the $250 Kansas food sales tax credit in the refundable-credit total. Kansas treats this credit as nonrefundable, leaving state_refundable_credits at $0." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model explicitly calculated approximately $2,173 from the correct taxable income and brackets, then submitted $2,540 instead. Its final value does not follow its own arithmetic." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the separately listed $21,208 employer-sponsored insurance premium from gross wages, reducing AGI to $15,068. It also incorrectly used the refundable EITC to reduce an output defined before refundable credits; the correct taxable income is $20,175.59 and the liability is $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model recalculated the liability as approximately $2,173 using the correct $16,100 deduction and rate brackets, then submitted $1,631. The submitted value contradicts its stated computation." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly computed approximately $2,173 from $20,176 of taxable income, then submitted $1,387 without any intervening deduction or credit. The final value is an unsupported substitution for its own result." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's stated bracket calculation yields approximately $2,173, but it replaced that amount with $2,029 under the label of bracket rounding. Rounding cannot account for the $144 reduction, and no credit applies." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an obsolete estimated $15,350 standard deduction and an $11,925 first-bracket ceiling instead of the 2026 values used in the calculation. The $16,100 deduction leaves $20,175.59 taxable, producing $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's own estimated deduction and brackets produced $2,226, but it submitted $1,130 with no credit or calculation supporting the reduction. The correct deduction and rate calculation yields $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted the $21,208 employer-sponsored insurance premium from the listed gross wages and applied a personal exemption plus post-sunset deduction rules. For 2026, AGI remains $36,275.59 and the applicable $16,100 standard deduction leaves $20,175.59 taxable." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the separately reported employer-sponsored insurance premium as a deduction from gross wages, lowering AGI to $15,068. The listed wages flow into AGI without that subtraction, so taxable income is $20,175.59 rather than zero." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model applied a head-of-household deduction even though this unmarried filer has no qualifying dependent and files single. It also failed to tax the income remaining after the deduction; the single-filer calculation yields $20,175.59 taxable and $2,173.07 of tax." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied pre-TCJA personal-exemption and 15% bracket rules instead of the operative 2026 $16,100 standard deduction and 10%/12% brackets. Those operative rules produce $20,175.59 of taxable income and $2,173.07 of tax." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $21,208 employer-sponsored insurance premium from gross wages and then used a personal exemption. The correct AGI is $36,275.59, followed only by the $16,100 standard deduction, leaving $20,175.59 taxable." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced wages by the employer-sponsored insurance premium and applied expired-law standard-deduction and personal-exemption amounts. The correct computation uses $36,275.59 of AGI and a $16,100 standard deduction, not $1,618 of taxable income." +us,scenario_045,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used estimated values of $15,400 for the standard deduction and $12,250 for the 10% bracket ceiling. Using the applicable $16,100 deduction and exact 2026 brackets produces $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction and unspecified nonrefundable credits as eliminating the liability. The $16,100 deduction leaves $20,175.59 taxable, and no facts support any nonrefundable credit, so $2,173.07 remains." +us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed unspecified nonrefundable credits fully offset the tax despite the instruction that unlisted inputs are zero. After the $16,100 standard deduction, $20,175.59 remains taxable and no nonrefundable credit reduces the resulting $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an estimated $15,750 standard deduction rather than the applicable $16,100 amount. That overstated taxable income and produced $2,215 instead of the exact $2,173.07 liability." +us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the $21,208 employer-sponsored insurance premium from the annual gross-wage input. AGI is $36,275.59, so the $16,100 standard deduction leaves $20,175.59 taxable rather than zero." +us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an approximate $15,000 standard deduction instead of $16,100 and did not apply the exact 2026 bracket parameters. The correct taxable income is $20,175.59 and the resulting tax is $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the employer-sponsored insurance premium from gross wages and then applied post-TCJA-sunset personal-exemption rules. The correct calculation retains $36,275.59 as AGI and subtracts the $16,100 standard deduction, yielding $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset, using a personal exemption, an approximately $8,000 standard deduction, and a 15% second bracket. The operative 2026 calculation uses the $16,100 standard deduction and main rates that produce $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. +us,scenario_045,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model correctly found about $20,176 of taxable income but then treated EITC and unspecified credits as eliminating the liability. EITC is refundable and excluded from this before-refundable-credits output, while no nonrefundable credit applies, leaving $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's written bracket calculation produced $2,220.12, but it submitted $424.76 despite stating that no nonrefundable credits apply. The submitted number is disconnected from its own derivation; using the exact $16,100 deduction yields $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model declared taxable income zero even though child support is nondeductible and the $16,100 standard deduction is far below $36,275.59 of AGI. The correct subtraction leaves $20,175.59 taxable, and no nonrefundable credit eliminates the $2,173.07 liability." +us,scenario_045,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly identified the childless EITC pathway and that its phaseout ends near $20,000, but then awarded $632 at $36,276. The childless EITC is fully phased out at that income, so this component is $0." +us,scenario_045,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted $21,208 of employer-sponsored insurance premiums from wages when computing EITC income, despite no stated pre-tax payroll treatment. It also produced $2,410 for a childless EITC whose maximum is far below that amount; using the full $36,276 leaves the credit fully phased out." +us,scenario_045,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model treated earned income as $15,068 by subtracting the $21,208 insurance premium from $36,276 of gross wages. The EITC test uses the full $36,276 here, at which the childless credit is fully phased out." +us,scenario_045,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model invented dependent status even though the household contains only one adult and all unlisted household facts are false. With no qualifying child and $36,276 of income, the childless EITC is fully phased out rather than $1,500." +us,scenario_045,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced earned income to $15,068 by treating the listed insurance premiums as a pre-tax wage deduction. No pre-tax treatment is stated, so EITC income remains $36,276 and the childless credit is fully phased out." +us,scenario_045,federal_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $15,068 as both AGI and earned income after subtracting employer-sponsored insurance premiums from gross wages. The full $36,276 enters the childless EITC test and produces no credit." +us,scenario_045,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model explicitly subtracted $21,208 of insurance premiums from gross wages as though they were pre-tax, yielding $15,068 of EITC income. Because that pre-tax treatment was not supplied, income remains $36,276 and the childless EITC is fully phased out." +us,scenario_045,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model computed the childless EITC from $15,068 after implicitly subtracting the insurance premiums from gross wages. The premiums are not a stated pre-tax wage exclusion, so the applicable income is $36,276 and the EITC is $0." +us,scenario_045,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied neither the required numeric output nor an explanation, so its response failed the submission contract." +us,scenario_045,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented one qualifying child even though only a single adult is listed and unlisted household members and statuses are false. The filer must use the no-qualifying-child EITC schedule, under which $36,276 is above the full phaseout range and the credit is $0." us,scenario_045,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_045,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_045,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the full $21,208 employer-sponsored insurance premium from $36,276 of wages to produce $15,068 of MAGI. The prompt does not identify those premiums as a pre-tax payroll deduction, so MAGI is approximately $36,276, or 2.27 times FPL, above Michigan's expansion limit." -us,scenario_045,head_medicaid_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced wages by the entire employer-sponsored insurance premium, yielding $15,068 of MAGI and false Healthy Michigan Plan eligibility. MAGI is approximately $36,276, or 2.27 times FPL, so the expansion-adult pathway fails." -us,scenario_045,head_medicaid_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model treated the $21,208 insurance premium as fully deductible from Medicaid MAGI and compared the resulting $15,068 with the 138% FPL limit. The supplied premium is not identified as a pre-tax deduction; MAGI remains approximately $36,276 and exceeds the limit." -us,scenario_045,head_medicaid_eligible,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model derived $15,068 by subtracting employer-sponsored insurance premiums from wages, then applied Michigan's 138% FPL expansion threshold. Medicaid MAGI is approximately $36,276, equal to 2.27 times FPL, so that pathway does not confer eligibility." -us,scenario_045,head_medicaid_eligible,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model understated MAGI at $15,068 by treating the full employer-sponsored insurance premium as a pre-tax income reduction. With MAGI of approximately $36,276, the head exceeds Michigan's adult-expansion limit and qualifies through no other category." -us,scenario_045,head_medicaid_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model explicitly classified all $21,208 of premiums as pre-tax and deducted them from wages. The facts establish employer-sponsored coverage but not a pre-tax MAGI deduction, leaving approximately $36,276 of MAGI, or 2.27 times FPL, above Michigan's expansion threshold." -us,scenario_045,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model correctly reasoned that income exceeds the expansion limit and even concluded Medicaid eligibility is 0, but then reversed that conclusion solely by asserting that PolicyEngine sets the result to 1. The head has no qualifying Medicaid category, receives no SSI, and is not a dependent, so no alternative pathway overrides the failed income test." +us,scenario_045,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the full $21,208 employer-sponsored insurance premium from $36,276 of wages to produce $15,068 of MAGI. The stated premium does not reduce Medicaid MAGI on its own, so MAGI remains $36,276 and exceeds Michigan's expansion threshold." +us,scenario_045,head_medicaid_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $15,068 as MAGI by deducting the entire $21,208 employer-sponsored insurance premium from wages. With MAGI of $36,276, the head is at 2.27 times FPL and is ineligible for the Healthy Michigan Plan." +us,scenario_045,head_medicaid_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced wages by the employer-sponsored insurance premium and compared the resulting $15,068 to the 138% FPL limit. The premium input does not establish a MAGI deduction, leaving $36,276 of MAGI and no expansion-adult eligibility." +us,scenario_045,head_medicaid_eligible,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated the full employer-sponsored insurance premium as a pre-tax MAGI deduction, yielding $15,068. Medicaid MAGI remains $36,276, which is 2.27 times FPL and above Michigan's adult expansion limit." +us,scenario_045,head_medicaid_eligible,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model derived estimated MAGI of $15,068 by subtracting the $21,208 insurance premium from wages without a valid MAGI deduction. The correct MAGI computation retains $36,276, so the expansion-adult pathway fails." +us,scenario_045,head_medicaid_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model explicitly deducted the full employer-sponsored insurance premium as pre-tax, reducing MAGI from $36,276 to $15,068. The household facts do not make that premium a Medicaid MAGI deduction, and MAGI of $36,276 exceeds 138% FPL." +us,scenario_045,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model correctly recognized that the head's income exceeds the expansion threshold but then asserted eligibility through unspecified other pathways and submitted 1 despite its own conclusion of 0. The 44-year-old head has no Medicaid eligibility category, receives no SSI, and qualifies through none of Michigan's alternative pathways." us,scenario_045,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_045,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $21,208 employer-sponsored insurance premium from wages and applied FICA to only $15,068. FICA applies to the full $36,276 wage base here, producing $2,775.08." -us,scenario_045,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed employer-sponsored insurance premium as an employee pre-tax payroll deduction, reducing FICA wages from $36,276 to $15,068. The premium does not reduce the payroll-tax base, so Social Security and Medicare taxes total $2,775.08." -us,scenario_045,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model excluded the entire $21,208 insurance premium from FICA wages without an input establishing a pre-tax employee salary reduction. Applying 6.2% Social Security and 1.45% Medicare rates to the applicable full wages yields $2,775.08." -us,scenario_045,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model incorrectly assumed the employer-sponsored insurance premium was a pre-tax employee contribution that reduced payroll wages to $15,068. The applicable FICA wage base remains $36,276, yielding $2,775.08." -us,scenario_045,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model identified the correct full-wage FICA method but failed to execute or transmit its arithmetic consistently: 7.65% of $36,276 is about $2,775, not $2,892. The trace components are $2,249.09 of Social Security tax and $526.00 of Medicare tax." -us,scenario_045,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"The model improperly deducted the $21,208 employer-sponsored insurance premium from FICA wages and taxed only $15,068. The listed premium does not establish a pre-tax employee payroll deduction, leaving $36,276 subject to employee Social Security and Medicare taxes." -us,scenario_045,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages to $15,068 by subtracting the employer-sponsored insurance premium. With no employee pre-tax payroll deduction specified, the full $36,276 is subject to FICA and produces $2,775.08." -us,scenario_045,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, violating the required output contract." -us,scenario_045,payroll_tax,minimax-m3,llm_error,other,False,"The model's stated component calculation correctly reaches approximately $2,775, but it submitted $2,837 instead. Its final value does not follow from its own $2,249 Social Security and $526 Medicare components; the engine-trace total is $2,775.08." +us,scenario_045,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the $21,208 employer-sponsored insurance premium from wages and applied FICA to only $15,068. Applying 6.2% Social Security and 1.45% Medicare rates to the full $36,276 yields $2,775.08." +us,scenario_045,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed employer-sponsored insurance premiums as automatically excluded from FICA wages, reducing the base from $36,276 to $15,068. The full $36,276 remains subject to employee Social Security and Medicare taxes." +us,scenario_045,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly classified the $21,208 insurance premium as a pre-tax FICA exclusion and computed payroll tax on $15,068. Payroll tax instead applies to the full $36,276, producing $2,249.09 of Social Security tax and $526.00 of Medicare tax." +us,scenario_045,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model assumed that the employer-sponsored insurance premium reduces payroll-taxable wages, although the input does not specify a qualifying employee salary-reduction contribution. Applying the 7.65% combined FICA rate to the full wage amount yields $2,775.08." +us,scenario_045,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model identified the correct $36,276 payroll tax base and the correct 6.2% and 1.45% rates but submitted $2,892, contradicting its own formula. Those inputs produce $2,249.09 of Social Security tax plus $526.00 of Medicare tax, totaling $2,775.08." +us,scenario_045,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"The model improperly deducted $21,208 of employer insurance premiums from FICA wages and taxed only $15,068. The full $36,276 is the applicable payroll tax base, yielding $2,775.08." +us,scenario_045,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model incorrectly treated the insurance premium as a pre-tax exclusion from FICA wages and reduced the base to $15,068. Social Security and Medicare taxes apply to the full $36,276, with no Michigan employee payroll tax or Additional Medicare Tax added." +us,scenario_045,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, violating the required output contract." +us,scenario_045,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly derived approximately $2,249 of Social Security tax and $526 of Medicare tax and even stated a total of approximately $2,775, but submitted $2,837. Its final numeric output does not follow its own component calculation; the unrounded components total $2,775.08." us,scenario_045,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_045,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated Michigan's 200% FPL BBCE gross-income screen as disqualifying and computed net income as $1,776. Categorical eligibility applies, and the engine's earned-income, standard, child-support, and shelter calculations produce $1,102.06 of net income, followed by the minimum-allotment floor." -us,scenario_045,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the full $21,208 employer-sponsored insurance premium from SNAP income and then invented a $258 monthly benefit. The correct SNAP deductions yield $1,102.06 monthly net income, and maximum allotment minus the 30% contribution falls below zero, so the minimum allotment applies." -us,scenario_045,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model oscillated between gross-income disqualification, a minimum allotment, and an unsupported $2,718 estimate. Michigan categorical eligibility and the correct deduction sequence yield $1,102.06 net income, after which the benefit is the minimum allotment, totaling $287.68 annually." -us,scenario_045,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model concluded that net income remained above the one-person limit without completing the applicable deduction calculation. The correct calculation produces $1,102.06 of net income, below $1,304.17, and then applies the minimum-allotment floor." -us,scenario_045,snap,claude-opus-5,llm_error,categorical_eligibility,False,The model stopped at a 200% FPL gross-income comparison. It missed categorical eligibility through TANF non-cash assistance and never performed the net-income and minimum-allotment calculations. -us,scenario_045,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income test as an absolute bar. Michigan categorical eligibility through TANF non-cash assistance controls, and the household passes the separate net-income test at $1,102.06." -us,scenario_045,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model calculated net income near $1,069 and then incorrectly converted eligibility near the benefit floor into a zero allotment. For an eligible one-person household, the minimum allotment applies when maximum allotment minus 30% of net income is nonpositive, producing $287.68 annually." -us,scenario_045,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model used the ordinary 130% FPL gross-income limit and stopped. It omitted Michigan categorical eligibility through TANF non-cash assistance, the net-income test, and the minimum-allotment floor." -us,scenario_045,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated Michigan's 200% FPL gross-income screen as dispositive. Categorical eligibility through TANF non-cash assistance applies, and the household passes the net-income test before receiving the minimum allotment." -us,scenario_045,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer reflects a shortcut from annual wages to income ineligibility. The model omitted categorical eligibility and the deductions that reduce SNAP net income to $1,102.06, as well as the minimum-allotment floor." -us,scenario_045,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model accepted that the gross test was passed but incorrectly concluded that deducted net income was too high. The correct deduction sequence yields $1,102.06, below the $1,304.17 net-income limit, and eligibility triggers the minimum allotment." -us,scenario_045,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model treated gross annual wages as exceeding an absolute eligibility limit. It missed Michigan categorical eligibility through TANF non-cash assistance and did not calculate the qualifying $1,102.06 net income." -us,scenario_045,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model incorrectly made Michigan's 200% FPL gross-income threshold dispositive. Categorical eligibility applies, and the subsequent net-income and minimum-allotment calculations produce a positive benefit." -us,scenario_045,snap,glm-5.2,llm_error,categorical_eligibility,False,The model applied the standard 130% FPL gross-income test and ended the calculation. It omitted categorical eligibility through TANF non-cash assistance and the qualifying net-income calculation. -us,scenario_045,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model inferred ineligibility directly from wages without calculating SNAP net income. The applicable deductions reduce net income to $1,102.06, below the limit, and the minimum allotment then applies." -us,scenario_045,snap,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The model incorrectly claimed that qualifying status and household size were missing even though the prompt specifies one person and a single household group. It therefore failed to evaluate categorical eligibility, the net-income test, and the minimum allotment." -us,scenario_045,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the non-elderly gross-income limit as an absolute bar. Michigan categorical eligibility through TANF non-cash assistance applies, after which the household passes the net-income test and receives the minimum allotment." -us,scenario_045,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model stopped at an annual-wage threshold comparison. It omitted categorical eligibility and the deduction calculation that produces qualifying net income of $1,102.06." -us,scenario_045,snap,gpt-5.6-sol,llm_error,thresholds_rates,False,The model concluded that the formula produced no positive allotment after deductions but failed to apply the statutory minimum-allotment floor for an eligible one-person household. That floor produces $287.68 across the year. -us,scenario_045,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model treated annual wages above a one-person income limit as automatically disqualifying. It missed categorical eligibility through TANF non-cash assistance and never reached the qualifying net-income and minimum-allotment computations. -us,scenario_045,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model asserted that income remained above the net limit without applying the full deduction calculation. Correct SNAP net income is $1,102.06 against a $1,304.17 limit, so the household qualifies and receives the minimum allotment." -us,scenario_045,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income test as a conclusive screen. Michigan categorical eligibility through TANF non-cash assistance bypasses that conclusion, and the household passes the net-income test." -us,scenario_045,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model relied on the 130% FPL gross-income test for a non-elderly, nondisabled person. It omitted Michigan categorical eligibility and the deductions that reduce net income below the applicable limit." -us,scenario_045,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, violating the required structured-output contract." -us,scenario_045,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model treated the 200% FPL BBCE gross-income screen as an absolute bar and deferred child support exclusively to a later deduction stage. The engine applies categorical eligibility through TANF non-cash assistance and then finds $1,102.06 of qualifying net income." -us,scenario_045,snap,minimax-m3,llm_error,categorical_eligibility,False,The model applied an approximate 130% FPL gross-income limit and stopped. It missed Michigan categorical eligibility through TANF non-cash assistance and the subsequent qualifying net-income calculation. -us,scenario_045,snap,qwen-3.7-max,llm_error,other,False,"The model's explanation concluded that SNAP was zero, but it submitted $7,032, so its numeric output contradicts its own derivation and the required answer contract. Its underlying reasoning also incorrectly treated the 130% FPL gross-income test as dispositive instead of applying categorical eligibility and the minimum allotment." +us,scenario_045,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated Michigan's 200% FPL gross-income screen as dispositive and computed net income without the deduction treatment that reduces it to $1,102.06. TANF non-cash categorical eligibility preserves eligibility, and the negative preliminary allotment is raised to the minimum allotment." +us,scenario_045,snap,claude-haiku-4.5,llm_error,other,False,"The model incorrectly deducted the entire $21,208 employer-sponsored insurance premium and then treated 30% of net income as the benefit rather than the expected contribution. SNAP equals the maximum allotment minus that contribution, subject here to the minimum-allotment floor, producing $287.68 annually rather than $3,100." +us,scenario_045,snap,claude-opus-4.7,llm_error,other,False,"The model abandoned its own negative-allotment calculation and substituted an unsupported $2,718 estimate. Categorical eligibility and the correct deductions yield $1,102.06 net monthly income, after which the minimum-allotment floor—not a larger shelter-based estimate—determines the $287.68 annual benefit." +us,scenario_045,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model concluded that the net-income test failed, but the applicable deductions produce $1,102.06, below the $1,304.17 limit. With TANF non-cash categorical eligibility, the household qualifies and receives the minimum allotment." +us,scenario_045,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model stopped at a 200% FPL gross-income screen. Michigan TANF non-cash categorical eligibility bypasses that dispositive shortcut, and the household passes the net-income test before receiving the minimum allotment." +us,scenario_045,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income test despite Michigan TANF non-cash categorical eligibility. The correct determination proceeds to net income, where $1,102.06 passes the limit and triggers the minimum allotment." +us,scenario_045,snap,claude-sonnet-5,llm_error,other,False,"The model's own deduction calculation reached net income below the eligibility threshold, then incorrectly reversed that result and returned zero. Because the household qualifies categorically and passes net income, a negative preliminary allotment is floored at the positive minimum rather than rounded to zero." +us,scenario_045,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated the ordinary 130% FPL gross-income limit as controlling. TANF non-cash categorical eligibility requires continuing to the net-income and benefit calculations, which produce the minimum allotment." +us,scenario_045,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model ended the analysis at Michigan's 200% FPL gross-income screen. The household is categorically eligible through TANF non-cash assistance and passes the $1,304.17 net-income limit with $1,102.06." +us,scenario_045,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer treats income as disqualifying without applying TANF non-cash categorical eligibility and the SNAP deductions. Those rules leave $1,102.06 of net monthly income and require the positive minimum allotment." +us,scenario_045,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model correctly recognized that the gross-income stage was passed but overstated SNAP net income after the earned-income, standard, child-support, and shelter calculations. The correct net amount is $1,102.06, below the $1,304.17 limit, and eligibility leads to the minimum allotment." +us,scenario_045,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model treated gross annual wages as an automatic bar. Michigan TANF non-cash categorical eligibility prevents that shortcut, and the household passes the net-income test after deductions." +us,scenario_045,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model incorrectly made Michigan's 200% FPL gross-income threshold dispositive. Categorical eligibility through TANF non-cash assistance carries the household into the net-income calculation and minimum-allotment floor. +us,scenario_045,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income test and stopped. TANF non-cash categorical eligibility applies, and the correctly deducted net income passes the relevant test." +us,scenario_045,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The zero answer implies that wages remained too high after SNAP rules, but the applicable deductions yield $1,102.06 of monthly net income. That amount passes the net-income test, and the minimum-allotment rule produces a positive benefit." +us,scenario_045,snap,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The model incorrectly claimed that household size and qualifying facts were missing even though the prompt specifies one head and states that all listed people form one household group. Applying those supplied facts gives categorical eligibility, a passed net-income test, and a positive minimum allotment." +us,scenario_045,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the non-elderly gross-income limit as dispositive. Michigan TANF non-cash categorical eligibility applies, after which deductions reduce net income below the limit and the minimum allotment is payable." +us,scenario_045,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model stopped at an unspecified Michigan income threshold. It omitted the TANF non-cash categorical-eligibility pathway and the net-income deductions that establish eligibility. +us,scenario_045,snap,gpt-5.6-sol,llm_error,other,False,"The model treated a nonpositive maximum-allotment-minus-contribution result as zero. For this eligible one-person household, SNAP applies the minimum-allotment floor, producing $287.68 annually." +us,scenario_045,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model treated annual wages above the ordinary one-person limit as disqualifying. TANF non-cash categorical eligibility and the subsequent net-income calculation establish eligibility for the minimum allotment. +us,scenario_045,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model overstated net income after SNAP deductions. PolicyEngine calculates $1,102.06, which is below the $1,304.17 net-income limit and leads to the minimum allotment." +us,scenario_045,snap,grok-4.5,llm_error,categorical_eligibility,False,The model incorrectly applied the ordinary 130% FPL gross-income screen as an automatic denial. Michigan TANF non-cash categorical eligibility requires proceeding to the passing net-income test and positive minimum allotment. +us,scenario_045,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model relied on the ordinary 130% FPL screen because the head is neither elderly nor disabled. It omitted Michigan's TANF non-cash categorical-eligibility pathway, which applies independently of elderly or disabled status." +us,scenario_045,snap,inkling,llm_error,categorical_eligibility,False,"The model treated the 130% FPL gross-income test as insurmountable and asserted that deductions could not matter. Categorical eligibility bypasses that shortcut, and the deduction structure reduces net income to $1,102.06." +us,scenario_045,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP output or explanation, so it failed the required output contract." +us,scenario_045,snap,kimi-k3,llm_error,categorical_eligibility,False,The model treated the 200% FPL BBCE screen as a final bar and excluded child support from the relevant path too early. TANF non-cash categorical eligibility and the full deduction calculation yield passing net income and the minimum allotment. +us,scenario_045,snap,minimax-m3,llm_error,categorical_eligibility,False,The model applied an approximate 130% FPL gross-income limit as dispositive. It omitted categorical eligibility through TANF non-cash assistance and therefore never reached the passing net-income calculation. +us,scenario_045,snap,qwen-3.7-max,llm_error,other,False,"The model's explanation concludes that SNAP is zero while its submitted numeric value is $7,032, violating the required same-variable value/explanation contract. It also applied the ordinary 130% FPL gross-income screen instead of the categorical-eligibility pathway." +us,scenario_045,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that net income exceeded the threshold, but the correct deduction calculation yields $1,102.06 against a $1,304.17 limit. The household therefore qualifies and receives the minimum allotment." us,scenario_045,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model substituted several speculative exemption amounts for Michigan's $5,950 personal exemption and then submitted $1,229 despite calculating figures between $1,272 and $1,291. Applying 4.25% to $36,275.59 minus $5,950 yields $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the separately listed $21,208 employer-sponsored insurance premiums from the stated annual wages and then applied a federal-style standard deduction that Michigan does not use here. It also treated the refundable Homestead Property Tax Credit as a nonrefundable offset, instead of taxing $30,325.59 after the $5,950 personal exemption." -us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $5,800 personal exemption instead of Michigan's $5,950 exemption. That left taxable income $150 too high and produced $1,295 rather than $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's reasoning calculated $1,295 from an incorrect $5,800 exemption, but its submitted value was an unsupported $1,024. The required derivation uses AGI of $36,275.59, a $5,950 exemption, and the 4.25% rate." -us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model first used the wrong $5,800 exemption and then invented unspecified nonrefundable credits to reduce its estimated $1,295 tax to $1,235. No applicable nonrefundable credit reduces the traced $1,288.84 liability." -us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied a 4.05% rate instead of Michigan's 4.25% rate and used a projected $6,000 exemption instead of $5,950. The correct taxable income is $30,325.59 and the resulting tax is $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model used a $5,800 exemption and correctly computed approximately $1,295 under that assumption, then submitted an unsupported $1,490. Michigan's $5,950 exemption and 4.25% rate produce $1,288.84, with no rounding adjustment that raises the result." -us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced wages by the separately listed $21,208 employer-sponsored insurance premiums, incorrectly setting federal AGI to $15,068. The wages already supply the income used by the trace, so AGI is $36,275.59 and only the $5,950 Michigan personal exemption is subtracted." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $15,068 as AGI by subtracting employer-sponsored insurance premiums from the listed wages, and it also used a $5,600 exemption instead of $5,950. Michigan taxable income is $30,325.59, not $9,468." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $1,150 estimate does not apply the specified Michigan computation. AGI of $36,275.59 minus the $5,950 exemption equals $30,325.59, and 4.25% of that amount is $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $5,800 state exemption rather than Michigan's $5,950 personal exemption. This overstated taxable income by $150 and tax by about $6.38." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the listed wages by employer-sponsored insurance premiums to obtain $15,068 of AGI, then used a $5,600 exemption. The trace uses $36,275.59 of AGI and a $5,950 exemption." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the employer-sponsored insurance premium as an additional deduction from wages, producing $15,068 of AGI, and used the wrong $5,600 exemption. Correct Michigan taxable income is $30,325.59." -us,scenario_045,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented a $20,000 Michigan standard deduction and treated the Homestead Property Tax Credit as a nonrefundable offset against this output. Michigan instead subtracts the $5,950 personal exemption from $36,275.59 of AGI, while the refundable Homestead credit does not reduce tax before refundable credits; its submitted zero also contradicts its own $121 calculation." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model assumed Michigan deductions eliminated all taxable income without identifying any valid deduction that does so. The applicable $5,950 personal exemption leaves $30,325.59 taxable and $1,288.84 of tax." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model assumed unspecified standard deductions and nonrefundable credits fully offset the liability. Michigan's traced calculation has only the $5,950 personal exemption and no nonrefundable credit reducing the resulting $1,288.84 tax." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model estimated the personal exemption at $5,900 instead of using $5,950 and rounded the AGI to $36,276. The traced inputs yield taxable income of $30,325.59 and tax of $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a 4.05% rate instead of Michigan's 4.25% rate and a $5,800 exemption instead of $5,950. Both parameter errors lowered the result from $1,288.84 to $1,234.28." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly set federal AGI to $15,068 by subtracting employer-sponsored insurance premiums from the listed wages, then estimated the exemption at $5,900. The correct base is $36,275.59 minus $5,950." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used an assumed $5,900 exemption rather than the $5,950 Michigan personal exemption and rounded AGI to $36,276. The exact traced taxable income and tax are $30,325.59 and $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model asserted that a Michigan standard deduction and exemptions eliminated $36,276 of income, but no such deduction applies to this filer. The $5,950 personal exemption leaves $30,325.59 subject to the 4.25% tax." -us,scenario_045,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced wages by the employer-sponsored insurance premiums to obtain $15,068 of federal AGI and also used a projected $5,880 exemption. The trace uses $36,275.59 of AGI and the enacted $5,950 exemption." -us,scenario_045,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $5,000 personal exemption instead of Michigan's $5,950 exemption. This overstated taxable income by $950 and produced $1,329 rather than $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so the required structured result was missing." -us,scenario_045,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used a $6,000 personal exemption instead of $5,950 and rounded AGI to $36,276. This understated the tax by about $2.11 relative to the traced $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model calculated approximately $1,329 of tax using an incorrect $5,000 exemption, correctly noted that the Homestead credit is refundable and excluded, and then submitted zero without a computation supporting that result. The correct $5,950 exemption leaves $1,288.84 before refundable credits." -us,scenario_045,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model used a $5,600 exemption and calculated $1,303.73, but then submitted an unsupported $1,116.01. The correct exemption is $5,950, producing $30,325.59 of taxable income and $1,288.84 of tax." -us,scenario_045,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly restricted Michigan’s Homestead Property Tax Credit to homeowners. Renters qualify because 23% of rent is treated as property tax, yielding the $760.79 refundable credit." -us,scenario_045,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model’s written homestead calculation already produced approximately $760.79 with no Michigan EITC, but it submitted $884. It introduced an unsupported final adjustment that contradicts its own arithmetic." -us,scenario_045,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model omitted the Michigan Homestead Property Tax Credit and then assigned an unsupported $1,209 despite concluding that the EITC and Home Heating Credit were zero. Applying the renter homestead calculation produces $760.79." -us,scenario_045,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model considered only the Michigan EITC and Home Heating Credit. It omitted the refundable Homestead Property Tax Credit available to renters, which supplies the entire $760.79." -us,scenario_045,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model treated the absence of separately listed property tax as disqualifying. Michigan counts 23% of a renter’s $10,560 annual rent as $2,428.80 of property tax, producing a $760.79 Homestead Property Tax Credit." -us,scenario_045,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model correctly found zero federal and Michigan EITC, then invented a reduced federal EITC and submitted $1,089. The applicable refundable amount is instead the renter’s Michigan Homestead Property Tax Credit of $760.79." -us,scenario_045,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced total household resources to $15,068 by subtracting the employer insurance premium and also assigned a federal EITC at income above the childless limit. PolicyEngine uses $36,275.59 of household resources here, 23% of rent, and zero EITC, yielding $760.79." -us,scenario_045,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly used 23% of rent but incorrectly reduced household resources to $15,068. Using $36,275.59 of household resources produces the $1,267.98 exemption and a $760.79 credit." -us,scenario_045,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model overlooked the Michigan Homestead Property Tax Credit for renters. The listed rent and household resources trigger a refundable credit of $760.79. -us,scenario_045,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model equated zero Michigan EITC with zero total state refundable credits. It omitted the renter’s Homestead Property Tax Credit, which equals $760.79." -us,scenario_045,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model used 20% rather than 23% of rent, applied an unsupported household-resource calculation, and added a Michigan EITC even though the childless federal EITC is zero at this income. The correct homestead-only amount is $760.79." -us,scenario_045,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model incorrectly awarded a federal and Michigan EITC at $36,276 of childless earnings and omitted the Homestead Property Tax Credit. The EITC is zero, while the renter homestead credit is $760.79." -us,scenario_045,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model used 20% instead of 23% of rent and then submitted zero despite calculating a positive homestead credit. Michigan treats $2,428.80 as property tax here, and the resulting refundable credit is $760.79." -us,scenario_045,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,The model failed to identify Michigan’s refundable Homestead Property Tax Credit for renters. The supplied rent and household resources yield $760.79. -us,scenario_045,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly treated a dependent or another unlisted condition as necessary for every Michigan refundable credit. A childless renter can claim the Homestead Property Tax Credit, and this household receives $760.79." -us,scenario_045,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model recognized the renter homestead pathway but used 20% instead of Michigan’s 23% rent conversion. Countable property tax is $2,428.80, producing a $760.79 credit rather than $570.70." -us,scenario_045,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $15,068 of household resources and added a Michigan EITC at income above the childless federal EITC limit. Using $36,275.59 of household resources and zero EITC yields a $760.79 homestead credit." -us,scenario_045,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model incorrectly treated being childless as disqualifying for all Michigan refundable credits. The Homestead Property Tax Credit does not require a child, and the renter qualifies for $760.79." -us,scenario_045,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The model omitted Michigan’s renter Homestead Property Tax Credit. Treating 23% of rent as property tax and applying the household-resource exemption yields $760.79. -us,scenario_045,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model reduced household resources to $15,068, used a 3.5% resource percentage, and added a nonzero Michigan EITC. PolicyEngine uses $36,275.59 of resources, zero EITC, and the homestead formula that yields $760.79." -us,scenario_045,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model failed to apply Michigan’s refundable Homestead Property Tax Credit to the renter. The listed $10,560 rent generates $2,428.80 of countable property tax and a $760.79 credit." -us,scenario_045,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for state_refundable_credits. It therefore failed the required output contract. -us,scenario_045,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,The model used 20% instead of 23% of rent and a 3.5% resource charge rather than the applicable household-resource exemption. The correct renter homestead calculation yields $760.79. -us,scenario_045,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model disregarded the explicitly listed pre-subsidy annual rent and wrongly concluded that no rent was paid. That $10,560 rent supports a refundable Michigan Homestead Property Tax Credit of $760.79." -us,scenario_045,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model rejected the Homestead Property Tax Credit without applying Michigan’s renter formula. The listed rent and household resources satisfy the pathway and yield $760.79. -us,scenario_046,child1_medicaid_eligible,minimax-m3,llm_error,health_coverage,False,"The model treated the child as generically likely eligible based on being a dependent child and did not apply Oklahoma's 2026 child Medicaid income thresholds to the household's 2.79 FPL MAGI. That income exceeds every applicable child Medicaid pathway, leaving medicaid_category = NONE and child1_medicaid_eligible = 0." -us,scenario_046,child2_medicaid_eligible,minimax-m3,llm_error,health_coverage,False,"The model treated child Medicaid eligibility as a generic likely outcome for a dependent child instead of applying Oklahoma's child Medicaid income pathways to the household's 2.79 FPL MAGI. It skipped the threshold comparison and categorical screening that leave child2 in no Medicaid category, producing eligible when the correct pathway result is not eligible." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own derivation produced approximately $488 after the overtime deduction and $4,400 CTC, then submitted $3,465 without a computation supporting that substitution. Applying the exact 2026 parameters to that same method yields $499.71." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $14,838.71 qualified-overtime deduction and used a $28,700 standard deduction instead of $32,200. It also invented an AOTC despite zero listed qualified education expenses and used obsolete CTC amounts." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model omitted the qualified-overtime deduction and used a $4,000 CTC instead of $4,400, leading its written calculation to $2,777. It then submitted $4,669, which contradicts even its own stated derivation." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model never deducted the $14,838.71 of qualified overtime, leaving taxable income about $14,839 too high. It also submitted tentative tax without subtracting the $4,400 CTC, despite stating that nonrefundable credits must be applied." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model correctly identified the standard and overtime deductions but reported tentative tax before child credits. The requested variable subtracts the full $4,400 nonrefundable CTC, reducing $4,899.71 to $499.71." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the qualified-overtime deduction and used a $30,000 standard deduction rather than $32,200. It also used the obsolete $2,000-per-child CTC instead of the 2026 $2,200 amount." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's narrative computed tentative tax near $6,696 and described credits that would reduce it, but its unexplained $12,200 submission is incompatible with those figures. It also omitted the qualified-overtime deduction and invented an AOTC with no qualified expenses." -us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a pre-TCJA regime with personal exemptions and 10%/15% brackets instead of the governing 2026 rules. It also omitted the qualified-overtime deduction and allowed only $2,000 of CTC rather than $4,400." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the overtime premium to wages, subtracted employer insurance premiums from wages, and applied personal exemptions. Gross wages already include overtime, while the premium creates a separate $14,838.71 deduction; the model also used only $2,000 of CTC instead of $4,400." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies that the model omitted the $14,838.71 qualified-overtime deduction and did not consistently subtract the stated child credits. The correct deduction sequence leaves $44,964.29 taxable, followed by $4,400 of CTC against $4,899.71 of tentative tax." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly treated the listed employer-sponsored insurance premium as a pre-tax wage deduction and applied itemized deductions plus personal exemptions. It omitted the qualified-overtime deduction and used only $2,000 total CTC instead of $4,400." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted employer insurance premiums from wages, applied personal exemptions, and used itemized deductions instead of the $32,200 standard deduction plus $14,838.71 overtime deduction. It also invented a $1,500 nonrefundable AOTC despite zero qualified education expenses and allowed only $2,000 total CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The submitted amount does not reflect the required $32,200 standard deduction, $14,838.71 qualified-overtime deduction, and $4,400 CTC combination. Those inputs yield $499.71, not $1,858.40." -us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model assigned an AOTC even though unlisted qualified education expenses equal zero. The two-child CTC offsets $4,400 of $4,899.71 tentative tax, leaving $499.71 rather than zero." -us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The answer reflects tax computed without the $14,838.71 qualified-overtime deduction and without the full $4,400 CTC. It also invokes an AOTC even though no qualified education expenses were listed." -us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model capped the qualified-overtime deduction at $12,500 instead of deducting the traced $14,838.71. That leaves taxable income $2,535 too high; using the full deduction produces $4,899.71 tentative tax and $499.71 after CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model omitted the $14,838.71 qualified-overtime deduction and failed to apply the $4,400 CTC. It also invoked AOTC phaseout even though zero qualified education expenses make the credit zero before any phaseout analysis." -us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied personal exemptions and 10%/15% tax brackets from an inapplicable regime, while omitting the qualified-overtime deduction. It also limited the two-child CTC to $2,000 rather than $4,400." -us,scenario_046,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a pre-TCJA tax structure and only $2,000 of CTC for two children. Under the governing 2026 rules, taxable income is $44,964.29 and the two children generate $4,400 of nonrefundable CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_046,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The unexplained $9,999 answer omits the trace's controlling computation: $92,003 AGI minus $47,038.71 of deductions, followed by $4,400 of CTC against $4,899.71 of tentative tax. That sequence yields $499.71." -us,scenario_046,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $14,838.71 qualified-overtime deduction, double-counted health premiums in medical expenses, treated nondeductible over-the-counter costs as medical deductions, and invented an AOTC despite zero qualified education expenses. It also used a $4,000 CTC instead of the 2026 $4,400 amount and reversed the AOTC refundable/nonrefundable shares." -us,scenario_046,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model treated the refundable CTC cap as an automatic refund even though the household's federal tax liability absorbs the available CTC nonrefundably. It also derived a $3,400 cap but submitted $3,200, so its final number does not follow its own computation." -us,scenario_046,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model assumed sufficient qualified education expenses to maximize the AOTC despite the instruction that unlisted numeric inputs are zero. With no listed tuition or other qualified education expenses, the refundable AOTC is $0." -us,scenario_046,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model counted the child credits and a maximum AOTC refundable amount without applying the ordering and expense rules. The available CTC is absorbed by tax liability, and zero listed qualified education expenses produce zero AOTC, so neither component generates its claimed refundable amount." -us,scenario_046,federal_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model correctly concluded that EITC and refundable CTC are zero, but awarded the maximum refundable AOTC solely from student-status facts. Qualified education expenses are unlisted and therefore zero under the prompt, making the AOTC, including its refundable portion, zero." -us,scenario_046,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model inferred that satisfying the AOTC student conditions automatically yields the $1,000 refundable maximum. The credit must be computed from qualified education expenses, which are unlisted and fixed at zero by the prompt, so refundable AOTC is $0." -us,scenario_046,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model awarded 40% of the maximum AOTC without any qualified tuition or related expenses. Because those numeric expenses are unlisted, they equal zero and generate no refundable AOTC." -us,scenario_046,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model provided no value for federal_refundable_credits, violating the required output contract." -us,scenario_046,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no value for federal_refundable_credits, violating the required output contract." -us,scenario_046,federal_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly reasoned that refundable CTC and EITC are zero, but invented a $1,000 refundable AOTC despite zero listed qualified education expenses. It then submitted $1,500 even though its stated component total was $1,000, adding a further unsupported $500." +us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model never used the 2026 Michigan personal exemption of $5,950 consistently and submitted $1,229 despite its own calculations producing values between $1,272 and $1,291. Subtracting $5,950 from AGI of $36,275.59 and applying 4.25% yields $1,288.84." +us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model wrongly subtracted the separately listed employer-sponsored insurance premiums from the annual gross-wage input and then invented a Michigan standard deduction. It also treated the refundable Homestead Property Tax Credit as a nonrefundable offset, although this output is measured before refundable credits." +us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model used an estimated $5,800 personal exemption instead of Michigan's $5,950 exemption for 2026. That left taxable income $150 too high and produced $1,295 instead of $1,288.84." +us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's submitted $1,024 contradicts its own stated calculation and final intermediate result of $1,295. Independently, that intermediate calculation also used a $5,800 exemption instead of $5,950." +us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model used an estimated $5,800 exemption and then subtracted an unsupported $60 of unspecified nonrefundable credits. No applicable nonrefundable credit reduces this liability; the traced calculation is 4.25% of $30,325.59." +us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied a 4.05% rate instead of Michigan's 4.25% rate and used a $6,000 exemption instead of $5,950. Both incorrect parameters reduced its result below $1,288.84." +us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model calculated approximately $1,295 from its assumed inputs but then submitted $1,490 with no arithmetic or credit adjustment supporting the increase. Its underlying exemption was also $5,800 rather than $5,950." +us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced the $36,276 annual gross-wage input by the separately listed $21,208 employer insurance premium, producing an erroneous $15,068 AGI. PolicyEngine uses AGI of $36,275.59, then subtracts only the $5,950 Michigan personal exemption." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $15,068 as AGI, which reflects subtracting the employer-sponsored insurance premium from the annual gross-wage amount. It also used a $5,600 exemption instead of $5,950." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The submitted $1,150 does not result from Michigan's 4.25% tax on AGI of $36,275.59 after the $5,950 personal exemption. The correct taxable base is $30,325.59, producing $1,288.84." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model used a $5,800 personal exemption instead of the 2026 Michigan amount of $5,950. This overstated taxable income by $150 and the tax by about $6.16." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced annual gross wages by the separately listed employer insurance premium to obtain $15,068 of AGI. It compounded that error by applying a $5,600 exemption rather than $5,950." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $15,068 of AGI by subtracting the employer-sponsored insurance premium from the annual gross-wage input. It then used the wrong $5,600 Michigan personal exemption instead of $5,950." +us,scenario_045,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model invented a $20,000 Michigan standard deduction and treated the Homestead Property Tax Credit as a nonrefundable credit against this output. Michigan instead subtracts the $5,950 personal exemption from AGI, while the refundable homestead credit belongs outside tax before refundable credits; its submitted zero also contradicts its own $121 calculation." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly asserted that Michigan deductions eliminate taxable income. Michigan taxable income remains $30,325.59 after the $5,950 personal exemption, and the 4.25% tax is $1,288.84." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed unspecified deductions and nonrefundable credits fully offset Michigan tax. The applicable calculation contains only the $5,950 personal exemption before applying the 4.25% rate, with no nonrefundable credit reducing the result." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model used an estimated $5,900 exemption instead of Michigan's $5,950 exemption. It also used rounded wages rather than traced AGI of $36,275.59, yielding $1,290.98 rather than $1,288.84." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied a 4.05% rate instead of Michigan's 4.25% rate and used a $5,800 exemption instead of $5,950. The correct parameters produce $1,288.84." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used $15,068 as federal AGI, improperly subtracting the separately listed employer insurance premiums from annual gross wages. Michigan begins with AGI of $36,275.59 and subtracts the $5,950 exemption." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model used a $5,900 personal exemption instead of Michigan's $5,950 exemption and rounded AGI to $36,276. The traced taxable income is $30,325.59, not $30,376." +us,scenario_045,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model invented a Michigan standard deduction large enough to erase taxable income. Michigan instead allows the $5,950 personal exemption here, leaving $30,325.59 subject to the 4.25% tax." +us,scenario_045,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly reduced annual gross wages by the separately listed employer-sponsored insurance premium and used $15,068 as AGI. It also used a projected $5,880 exemption rather than the $5,950 Michigan amount." +us,scenario_045,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model used a $5,000 personal exemption instead of Michigan's $5,950 exemption for 2026. This overstated taxable income by about $950 and produced $1,329 rather than $1,288.84." +us,scenario_045,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model subtracted a Homestead Property Tax Credit from state tax before refundable credits. That credit is refundable and therefore belongs in the separate state_refundable_credits output, leaving the $1,288.84 pre-refundable liability intact." +us,scenario_045,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_045,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model used a $6,000 personal exemption rather than Michigan's $5,950 exemption. That reduced taxable income by an extra $50 and tax by about $2.11." +us,scenario_045,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model computed roughly $1,329 of tax and correctly recognized the homestead credit as refundable, but then submitted zero without any valid offset or arithmetic. It also used a $5,000 exemption instead of $5,950." +us,scenario_045,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's submitted $1,116.01 contradicts its repeated calculation of $1,303.73 and has no supporting computation. Its repeated calculation also used a $5,600 exemption instead of Michigan's $5,950 exemption." +us,scenario_045,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted both a nonexistent Michigan standard deduction and the nondeductible child-support payment to reduce taxable income to zero. Michigan instead subtracts only the $5,950 personal exemption from the traced AGI in this calculation." +us,scenario_045,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly restricted Michigan's Homestead Property Tax Credit to homeowners. Michigan renters qualify through the rent-as-property-tax calculation, which produces $760.79 here." +us,scenario_045,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's own calculation reached approximately $760.78 and correctly found no Michigan EITC, but it submitted $884 without any supporting computation. It failed to carry its calculated Homestead Property Tax Credit into the final value." +us,scenario_045,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model never calculated the Michigan Homestead Property Tax Credit from the listed rent and household resources, then asserted an unsupported $1,209 amount despite finding both the EITC and Home Heating Credit equal to zero. The applicable homestead calculation yields $760.79." +us,scenario_045,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model considered only the Michigan EITC and Home Heating Credit and omitted the refundable Homestead Property Tax Credit. The renter pathway based on $10,560 of rent produces $760.79." +us,scenario_045,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model treated the absence of separately listed property tax as disqualifying even though the prompt lists annual rent. Michigan converts a prescribed portion of renters' rent into countable property tax, producing a $760.79 Homestead Property Tax Credit." +us,scenario_045,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model correctly determined that the childless federal and Michigan EITCs are zero, then invented a reduced federal-EITC basis and submitted $1,089. It omitted the actual source of the refundable credit: the $760.79 Michigan Homestead Property Tax Credit." +us,scenario_045,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced household resources to $15,068, effectively subtracting the listed employer-sponsored insurance premiums from the stated annual wages, and also invented a positive childless EITC. Michigan uses household resources of $36,275.59 here, and the Homestead Property Tax Credit alone equals $760.79." +us,scenario_045,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used $15,068 instead of $36,275.59 as Michigan household resources, effectively subtracting employer-sponsored insurance premiums from the stated wages. That understated the resource-based offset and overstated the Homestead Property Tax Credit; the correct credit is $760.79." +us,scenario_045,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model omitted Michigan's refundable Homestead Property Tax Credit. The listed rent supplies the renter property-tax base, and the calculation yields $760.79." +us,scenario_045,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model treated the Michigan EITC as the only relevant refundable state credit. Although that EITC is zero, the renter qualifies for a $760.79 Michigan Homestead Property Tax Credit." +us,scenario_045,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model invented a positive federal and Michigan EITC and used the wrong rent percentage and household-resource treatment for the homestead credit. The EITCs are zero, and the Michigan Homestead Property Tax Credit computed from the traced rent and resources is $760.79." +us,scenario_045,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model assigned a positive federal EITC to a childless filer earning $36,276 even though that income is above the childless EITC limit, and it omitted the Homestead Property Tax Credit. The EITC contribution is zero and the homestead credit is $760.79." +us,scenario_045,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model used 20% rather than 23% of rent as countable property tax, which understated the Homestead Property Tax Credit, and then contradicted its own positive-credit computation by submitting zero. The applicable renter calculation yields $760.79." +us,scenario_045,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,The model failed to identify Michigan's refundable Homestead Property Tax Credit from the listed rent. The renter calculation produces $760.79. +us,scenario_045,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model wrongly assumed a dependent or another unlisted condition was required for all Michigan refundable credits. A childless renter can receive the Homestead Property Tax Credit, which equals $760.79 here." +us,scenario_045,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model correctly used the renter pathway but treated only 20% of rent as property tax. Michigan's applicable calculation starts with $2,428.80 of countable property tax from the $10,560 rent and yields $760.79." +us,scenario_045,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used $15,068 rather than $36,275.59 as household resources and added a positive Michigan EITC even though the childless federal EITC is zero at this income. The Homestead Property Tax Credit is the sole refundable state credit and equals $760.79." +us,scenario_045,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model incorrectly treated being childless as disqualifying for every Michigan refundable credit. The Homestead Property Tax Credit has a renter pathway that produces $760.79 without a qualifying child. +us,scenario_045,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The model omitted the Michigan Homestead Property Tax Credit despite the listed annual rent and qualifying household resources. That refundable credit equals $760.79. +us,scenario_045,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used $15,068 rather than $36,275.59 as household resources, applied the wrong resource percentage, and added a positive EITC at income above the childless limit. The homestead credit alone equals $760.79." +us,scenario_045,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model failed to apply Michigan's refundable Homestead Property Tax Credit to the renter. The listed $10,560 rent and qualifying household resources produce $760.79." +us,scenario_045,state_refundable_credits,inkling,llm_error,state_local_rule,False,The model incorrectly conditioned the Homestead Property Tax Credit's refundable amount on exceeding state income-tax liability. The homestead credit is refundable independently of that liability and equals $760.79 here. +us,scenario_045,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric value or explanation for state_refundable_credits, so the required output was missing." +us,scenario_045,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,The model used 20% rather than 23% of rent as countable property tax and applied an incorrect 3.5% resource offset. The applicable Michigan renter calculation produces $760.79. +us,scenario_045,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model disregarded the explicitly listed pre-subsidy annual rent and concluded that no rent was paid. That $10,560 rent supplies the renter property-tax base for a $760.79 Homestead Property Tax Credit." +us,scenario_045,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model rejected Homestead Property Tax Credit qualification without applying Michigan's renter pathway to the listed rent and household resources. That pathway yields $760.79. +us,scenario_045,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,The model omitted Michigan's refundable Homestead Property Tax Credit. The renter's listed rent and qualifying household resources produce a credit of $760.79. +us,scenario_046,child1_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model asserted that the child was “likely eligible” without testing Oklahoma's income limits or any categorical pathway. At 2.79 times FPL, child1 fails the regular children's Medicaid limit, CHIP limit, and all other applicable categories, yielding medicaid_category = NONE and value = 0." +us,scenario_046,child2_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model assumed that being a child made Child 2 eligible without testing Oklahoma's categorical and income requirements. At 2.79 times FPL, Child 2 exceeds all applicable child Medicaid limits and qualifies through no other pathway, so the correct value is 0." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its written calculation reaches approximately the required taxable income, tentative tax, and $4,400 CTC, but it then submits $3,465 instead of the roughly $488 it derived. The final value is an unsupported arithmetic/output substitution inconsistent with its own reasoning." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified-overtime deduction and used a $28,700 standard deduction instead of $32,200. It also invented an AOTC without listed qualified expenses and limited the CTC incorrectly rather than applying the full $4,400 against $4,899.71 of tentative tax." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It omitted the qualified-overtime deduction, used the wrong standard deduction, and applied a $4,000 CTC instead of $4,400. Its submitted $4,669 also contradicts the $2,777 result stated in its own derivation." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified-overtime deduction, leaving taxable income far too high. It then submitted essentially tentative tax without subtracting the available $4,400 CTC, despite stating that nonrefundable credits reduce the liability." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It substantially identified the correct deduction path but reported tentative tax before child credits instead of the requested tax after nonrefundable credits. Subtracting the $4,400 CTC from the exact $4,899.71 tentative tax yields $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It omitted the qualified-overtime deduction and used a $30,000 standard deduction, inflating taxable income to $62,003 instead of $44,964.29. It also used a $4,000 CTC rather than the 2026 amount of $4,400 for two children." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It omitted the qualified-overtime deduction in its calculation, invented an AOTC despite zero listed qualified expenses, and misunderstood how the CTC is divided between nonrefundable and refundable amounts. Its $12,200 submission follows none of its stated arithmetic and exceeds its own tentative tax." +us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It incorrectly applied pre-TCJA personal exemptions and 10%/15% brackets instead of the operative 2026 standard deduction and tax brackets. It also allowed only $2,000 of CTC instead of $4,400 and failed to use the $14,838.71 overtime deduction in the traced calculation." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It reconstructed wages incorrectly, subtracted the employer-sponsored insurance premium from income, and applied personal exemptions and itemized deductions instead of the $32,200 standard deduction plus $14,838.71 overtime deduction. It also used only $2,000 of CTC rather than $4,400." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"Its $8,518 result is inconsistent with its stated $92,003 AGI, $31,200 deduction, 2026 brackets, and $4,000 child credit. It also omitted the $14,838.71 overtime deduction and understated the CTC by $400." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It incorrectly treated employer-sponsored insurance premiums as a deduction from the already stated gross wages and applied personal exemptions under obsolete post-TCJA-expiration rules. It omitted the overtime deduction and allowed only $2,000 of CTC instead of $4,400." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It improperly subtracted the ESI premium, applied personal exemptions and itemized deductions, and invented a $1,500 nonrefundable AOTC with no qualified education expenses listed. It also used only $2,000 of CTC instead of the two-child $4,400 credit." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The unexplained $1,858.40 does not reflect the traced sequence of $44,964.29 taxable income, $4,899.71 tentative tax, and $4,400 CTC. The answer therefore embodies an incorrect deduction, rate, or credit computation rather than the required calculation." +us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no numeric output for the requested variable, so the answer failed the required submission contract." +us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It invented an AOTC offset even though no qualified education expenses were listed and then defaulted the liability to zero. The actual nonrefundable credit is the $4,400 CTC, which reduces $4,899.71 to $499.71 rather than zero." +us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its $6,794 answer reflects tax near the pre-credit amount obtained after only a standard deduction, omitting the $14,838.71 qualified-overtime deduction. It also failed to subtract the $4,400 CTC and invoked an AOTC without listed qualified expenses." +us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It capped the qualified-overtime deduction at $12,500 even though the joint-filer cap permits the full $14,838.71. That understatement raised taxable income and tentative tax, producing $780.36 instead of $499.71 after the correctly identified $4,400 CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It supplied no computation supporting $8,500 and incorrectly invoked an AOTC phaseout even though no qualified education expenses exist. The required standard and overtime deductions yield $4,899.71 before the $4,400 CTC, not an $8,500 post-credit liability." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied personal exemptions and 10%/15% brackets under obsolete post-TCJA-expiration rules, while also treating health premiums as deductible medical expenses and omitting the traced standard-deduction path. It allowed only $2,000 of CTC rather than $4,400." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used pre-TCJA rules and an unexplained $23,508 taxable-income figure instead of $44,964.29 under the 2026 standard and overtime deductions. It also used only $2,000 of CTC rather than $4,400." +us,scenario_046,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted the overtime amount from AGI and then improperly itemized medical expenses, producing taxable income of about $34,706 instead of applying the $32,200 standard deduction alongside the overtime deduction to reach $44,964.29. This understated tentative tax enough for the $4,400 CTC to erase it, whereas the correct tentative tax exceeds the credit by $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no numeric output for the requested variable, so the answer failed the required submission contract." +us,scenario_046,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The unsupported $9,999 does not incorporate the traced $44,964.29 taxable income or the $4,400 nonrefundable CTC. Applying the 2026 rates to that taxable income gives $4,899.71 before credits and $499.71 after the CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified-overtime deduction, improperly counted health premiums and over-the-counter costs as itemized medical deductions, and invented an AOTC with no qualified education expenses. It also reversed the AOTC refundable split and used a $4,000 CTC instead of $4,400." +us,scenario_046,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It omitted the qualified-overtime deduction, invented a $2,500 AOTC with no qualified expenses, and understated the CTC at $4,000. Its submitted $8,838 is also wholly inconsistent with the $342 result stated in its own reasoning." +us,scenario_046,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model treated the refundable CTC earnings formula and per-child cap as an automatic refund even though the household’s tax liability absorbs the available CTC nonrefundably. Its stated calculation produces a $3,400 cap but it submitted $3,200, adding an internal arithmetic inconsistency." +us,scenario_046,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model invented sufficient qualified education expenses to maximize the AOTC despite the instruction that every unlisted numeric input is zero. Student eligibility facts without qualified expenses produce no AOTC and therefore no $1,000 refundable portion." +us,scenario_046,federal_refundable_credits,claude-opus-5,llm_error,other,False,"The model treated the $4,400 Child Tax Credit and a maximum AOTC as refundable amounts rather than separating credits used against liability from refundable portions. Qualified education expenses are zero, and the available CTC is absorbed nonrefundably, so neither produces refundable credits." +us,scenario_046,federal_refundable_credits,claude-sonnet-4.6,llm_error,other,False,"The model correctly concluded that the tax liability absorbs the Child Tax Credit, but then awarded the maximum refundable AOTC based only on enrollment and documentation status. Because no qualified education expenses are listed, the prompt sets the AOTC expense base to zero." +us,scenario_046,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model awarded a $1,000 refundable AOTC solely because the head satisfies student-status requirements. The missing qualified education expense input is zero under the prompt, so the AOTC and its refundable portion are both zero." +us,scenario_046,federal_refundable_credits,gemini-3.5-flash,llm_error,other,False,"The model assumed that AOTC eligibility automatically yields the $2,500 maximum credit. With no qualified tuition or related expenses listed, the credit base is zero and the 40% refundable portion is also zero." +us,scenario_046,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no output for federal_refundable_credits, so it failed the required structured-output contract." +us,scenario_046,federal_refundable_credits,inkling,llm_error,other,False,"The model understated the federal tax liability available to absorb the Child Tax Credit and therefore created a nonexistent $725 residual refundable CTC. The liability absorbs the available CTC nonrefundably, leaving refundable CTC of zero." +us,scenario_046,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output for federal_refundable_credits, so it failed the required structured-output contract." +us,scenario_046,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model invented qualified education expenses and awarded a $1,000 refundable AOTC even though unlisted expenses are zero. It then submitted $1,500 despite its own component calculation totaling $1,000, introducing an additional unexplained $500." us,scenario_046,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_046,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly derived $5,704 of Social Security tax and $1,334 of Medicare tax but added them incorrectly as $10,038 instead of $7,038. It then invented a $518.40 Oklahoma payroll-tax adjustment even though no mandatory Oklahoma employee payroll tax applies." -us,scenario_046,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly deducted the $9,208 employer-sponsored insurance premium from FICA wages. Payroll tax applies to the full $92,000 of listed wages, producing $7,038 rather than 7.65% of $82,792." -us,scenario_046,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model double-counted the $14,839 overtime premium by adding it to the $92,000 gross-wage total, even though the prompt states that gross wages already include overtime. It also subtracted the $9,208 employer-sponsored insurance premium without an employee pretax contribution being listed, creating the erroneous $97,631 FICA base." -us,scenario_046,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly treated the $9,208 employer-sponsored insurance premium as an employee pretax deduction from FICA wages. The applicable payroll-tax base is the full $92,000, not $82,792." -us,scenario_046,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly reduced FICA-taxable wages by the $9,208 employer-sponsored insurance premium. No employee pretax payroll contribution is listed, so Social Security and Medicare apply to the full $92,000." -us,scenario_046,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly subtracted the $9,208 employer-sponsored insurance premium from the payroll-tax base. Applying 7.65% to the full $92,000 yields $7,038." -us,scenario_046,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output, so it failed the required structured-output contract." -us,scenario_046,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model set the entire liability to zero merely because no state payroll-tax rate was specified, omitting the directly computable federal employee Social Security and Medicare taxes. Those federal components total $7,038, while Oklahoma contributes zero additional employee payroll tax." -us,scenario_046,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's $7,172 answer adds $134 beyond the standard $5,704 Social Security and $1,334 Medicare liabilities. No Additional Medicare Tax or mandatory Oklahoma employee payroll tax applies, so the computation stops at $7,038." -us,scenario_046,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"The model understated standard FICA on $92,000 by $10. At 6.2% Social Security and 1.45% Medicare, the components are exactly $5,704 and $1,334, totaling $7,038 rather than $7,028." -us,scenario_046,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly deducted the $9,208 employer-sponsored insurance premium from wages before applying FICA. The full $92,000 is subject to the standard employee Social Security and Medicare rates." -us,scenario_046,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output, so it failed the required structured-output contract." -us,scenario_046,payroll_tax,minimax-m3,llm_error,thresholds_rates,False,"The model did not apply the standard combined employee FICA rate of 7.65% to the stated $92,000 wage base. That calculation yields $7,038, not $7,193.10, and no additional federal or Oklahoma employee payroll tax applies." -us,scenario_046,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's reasoning correctly computed and totaled $5,704 of Social Security tax and $1,334 of Medicare tax as $7,038, but it submitted $8,438. The numeric output therefore contradicts its own completed derivation by $1,400." +us,scenario_046,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly derived $5,704 of Social Security tax and $1,334 of Medicare tax but misadded them as $10,038 instead of $7,038. It then invented a $518.40 Oklahoma employee payroll tax or “rounding adjustment,” even though no such liability applies." +us,scenario_046,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly deducted the $9,208 employer-sponsored insurance premium from FICA-taxable wages. Applying 7.65% to the full $92,000 wage base yields $7,038." +us,scenario_046,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model added the $14,839 overtime premium to the $92,000 gross-wage total even though the prompt states that gross wages already include overtime, then deducted the insurance premium. This produced an erroneous $97,631 payroll-tax base instead of $92,000." +us,scenario_046,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly reduced FICA-taxable wages from $92,000 to $82,792 by deducting the listed employer-sponsored insurance premium. Social Security and Medicare taxes are assessed on the full $92,000 base in this household." +us,scenario_046,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model treated the $9,208 employer-sponsored insurance premium as a pre-tax FICA deduction and taxed only $82,792. The applicable payroll-tax base is the full $92,000 of wages, producing $7,038." +us,scenario_046,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly subtracted the $9,208 insurance premium from wages before applying the 7.65% employee FICA rate. Applying that rate to $92,000 yields $7,038." +us,scenario_046,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, violating the required output contract." +us,scenario_046,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model set payroll tax to zero merely because no state payroll-tax rate was specified, ignoring the computable federal employee Social Security and Medicare taxes. Those federal components are $5,704 and $1,334, totaling $7,038." +us,scenario_046,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model failed to calculate the stated federal components and implicitly added $134 beyond the $7,038 due on $92,000 of wages. Oklahoma contributes no mandatory employee payroll tax for this result." +us,scenario_046,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model made a $10 arithmetic error when combining employee Social Security and Medicare taxes on $92,000. The components are $5,704 and $1,334, which sum to $7,038 rather than $7,028." +us,scenario_046,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly deducted $9,208 of health premiums from the FICA wage base and applied the tax rates to only $82,792. The computation uses the full $92,000 of wages and yields $7,038." +us,scenario_046,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model explicitly treated the employer-sponsored insurance premium as a pre-tax reduction of FICA wages, lowering the base to $82,792. The full $92,000 wage amount generates $5,704 of Social Security tax and $1,334 of Medicare tax." +us,scenario_046,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, violating the required output contract." +us,scenario_046,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model did not apply the standard 7.65% employee FICA rate correctly to the $92,000 wage base. That calculation is $5,704 of Social Security tax plus $1,334 of Medicare tax, not $7,193.10." +us,scenario_046,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's reasoning correctly calculated and totaled $5,704 of Social Security tax and $1,334 of Medicare tax as $7,038, but it submitted $8,438. The submitted numeric value contradicts its own completed computation." us,scenario_046,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_046,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model reached the correct $75,303 taxable income but overstated bracket tax as roughly $3,222 instead of $2,959.14 and used a $200 credit instead of $220. It then submitted $3,364 despite its own stated $3,022 result." -us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $14,600 deduction, an obsolete 5.85% top-rate approximation, and an unsupported lump-sum credit reduction. Oklahoma's joint brackets produce $2,959.14 on $75,303, followed by the $220 state credit." -us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly derived $75,303 of taxable income but overstated regular Oklahoma tax as about $3,221 and estimated the child credit rather than calculating $220 from the $4,400 federal CTC. The required subtraction is $2,959.14 minus $220." -us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model imposed a $17,000 itemized-deduction cap, reduced taxable income to $71,003, and omitted the $220 nonrefundable child credit. The applicable derivation uses $75,303 of Oklahoma taxable income, $2,959.14 of bracket tax, and the $220 credit." -us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model reached approximately the correct $75,303 taxable income but compressed the bracket and credit calculation into an erroneous $2,500 estimate. Applying the joint brackets and then the $220 credit yields $2,739.14." -us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly substituted an approximately $30,000 federal standard deduction for Oklahoma's applicable $12,700 deduction, reducing taxable income to $58,003. It also omitted the $220 Oklahoma child care/child tax credit." -us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model inconsistently considered both a federal-sized standard deduction and $20,879 of itemized deductions, then estimated tax on roughly $71,121 without completing the prescribed deduction calculation. The correct taxable income is $75,303, and the resulting $2,959.14 tax is reduced by the $220 credit." -us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced Oklahoma taxable income to $51,795.62 through an unsupported federal-itemized-deduction calculation. The applicable deductions and four exemptions leave $75,303 taxable, followed by $2,959.14 of bracket tax and a $220 credit." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model combined a $17,000 cap with a $5,430 medical deduction, producing the wrong $65,573 taxable income. It also calculated only a $100 state child credit instead of 5% of the $4,400 federal CTC, which is $220." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The unexplained $3,154 does not implement the traced result: $75,303 of taxable income produces $2,959.14 under the joint brackets, and the $220 nonrefundable credit reduces it to $2,739.14." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model's $2,105 estimate reflects an excessive itemized-deduction reduction and omits the explicit $220 credit computation. Oklahoma taxable income is $75,303, not the substantially lower base implicit in its answer." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced AGI to $82,795 and claimed $26,999 of itemized deductions, leaving only $51,796 taxable. It also used a $100 child credit rather than the $220 credit derived from the $4,400 federal CTC." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The submitted $2,276.01 implies that the model understated the taxable base or overstated deductions, while providing no supporting figures. The traced base is $75,303, whose bracket tax of $2,959.14 is reduced only by the $220 nonrefundable credit." -us,scenario_046,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model replaced a computable Oklahoma liability with a zero placeholder. The supplied facts determine $75,303 of taxable income, $2,959.14 of bracket tax, and a $220 nonrefundable credit." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model improperly invoked the federal American Opportunity Credit in calculating Oklahoma tax and did not show the applicable state computation. The state calculation instead subtracts the $220 Oklahoma child care/child tax credit from $2,959.14." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly derived $75,303 of taxable income and the $220 credit but calculated regular Oklahoma tax as $3,064 instead of $2,959.14. Correct application of the joint-filer brackets yields $2,739.14 after the credit." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The model's $4,223.29 does not follow from its claimed overtime deduction and supplies neither a taxable-income figure nor a valid bracket calculation. The traced computation produces $2,959.14 before the $220 credit, not a liability above $4,000." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model imposed a $17,000 capped itemized deduction and obtained $71,003 of taxable income instead of $75,303. Its associated $3,018 bracket estimate is also inconsistent with the applicable joint schedule; the correct pre-credit tax is $2,959.14." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model replaced Oklahoma's graduated joint-filer schedule with a 4.5% approximation and treated the state credit as two fixed $100 child credits. The applicable credit is $220, derived as 5% of the $4,400 federal CTC." -us,scenario_046,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The unexplained $3,800 overstates Oklahoma tax on the traced $75,303 taxable base and does not subtract the $220 nonrefundable credit. The joint brackets produce $2,959.14 before that credit." -us,scenario_046,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model deducted $26,309 of purported federal itemized deductions and then another $4,000 of exemptions, producing an incorrect $61,694 taxable base. It also stopped at bracket tax and omitted the $220 Oklahoma child care/child tax credit." -us,scenario_046,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model invented a $45,216 taxable base and applied a flat 4.5% rate. Oklahoma uses the traced $75,303 taxable income and graduated joint-filer brackets, followed by the $220 nonrefundable credit." -us,scenario_046,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_046,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the four $1,000 Oklahoma exemptions, leaving $79,303 rather than $75,303 taxable. It also asserted that no state credit applied, omitting the $220 Oklahoma child care/child tax credit." -us,scenario_046,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The unexplained $3,773 does not match the required derivation. Oklahoma's joint brackets produce $2,959.14 on $75,303, and the $220 nonrefundable credit reduces the requested amount to $2,739.14." -us,scenario_046,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model overstated AGI as approximately $94,004 despite the listed $92,000 wages and $3 interest, and it omitted all four $1,000 Oklahoma exemptions. These errors produced an inflated taxable base, and it also failed to subtract the $220 state credit." +us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model selected Oklahoma's $12,700 standard deduction after incorrectly declaring itemizing disadvantageous, instead of applying the $22,429.78 itemized-deduction computation. It also used a $200 credit rather than the $220 credit derived from the $4,400 federal CTC, then submitted $3,364 despite its own $3,022 calculation." +us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model treated Oklahoma tax as 5.85% of an incorrectly constructed federal-taxable-income base and then inserted unspecified credits to reach $1,614. Oklahoma requires its own deductions, exemptions, graduated joint-filer brackets, and the specific $220 nonrefundable child credit." +us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used the $12,700 standard deduction instead of the $22,429.78 itemized-deduction computation. It then approximated the child credit rather than applying the exact $220 reduction from $2,959.14 to $2,739.14." +us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model imposed a $17,000 cap as the entire relevant itemized deduction and constructed taxable income of $71,003, rather than using Oklahoma itemized deductions of $22,429.78 in the traced calculation. It also omitted the $220 nonrefundable child credit." +us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used the $12,700 standard deduction instead of the larger $22,429.78 itemized-deduction amount. Its rounded $2,500 estimate also failed to calculate the bracket tax of $2,959.14 and subtract the exact $220 nonrefundable credit." +us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that Oklahoma conforms to an approximately $30,000 federal joint standard deduction and used that amount instead of the traced $22,429.78 itemized deduction. It also omitted the $220 Oklahoma child care/child tax credit despite the household qualifying through its federal CTC." +us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model inconsistently claimed Oklahoma has no income tax and then estimated tax using only $20,879 of mortgage-interest and real-estate-tax deductions. It missed the full $22,429.78 itemized-deduction computation and did not correctly apply the $220 nonrefundable child credit." +us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to $51,795.62 through an unsupported federal-itemized-deduction calculation. The traced Oklahoma calculation uses $92,003 of AGI, $22,429.78 of itemized deductions, four exemptions, the Oklahoma brackets, and a $220 credit." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model combined a $17,000 cap with a separately calculated $5,430 medical deduction, producing the wrong deduction base and taxable income. It also calculated only a $100 child credit instead of 5% of the household's $4,400 federal CTC, which is $220." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The unexplained $3,154 result fails to reproduce the traced sequence: $22,429.78 of Oklahoma itemized deductions, four exemptions, $2,959.14 of bracket tax, and a $220 nonrefundable credit. Applying that sequence yields $2,739.14." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model states only that unspecified itemized deductions and exemptions produce $2,105, so it did not use the traced $22,429.78 deduction and resulting Oklahoma bracket calculation. It also omitted the $220 nonrefundable child credit from its stated derivation." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly lowered Oklahoma AGI to $82,795 and deducted $26,999, producing taxable income of $51,796. Oklahoma AGI is $92,003, the traced itemized deduction is $22,429.78, and the applicable child credit is $220 rather than $100." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The unexplained $2,276.01 total does not follow the traced computation of $2,959.14 before credits less the $220 Oklahoma child credit. Its answer therefore embeds an unsupported deduction, bracket, or credit reduction of $463.13 beyond the traced liability." +us,scenario_046,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model replaced a computable Oklahoma liability with a zero placeholder. The supplied facts determine $2,959.14 of tax under the Oklahoma calculation and a $220 nonrefundable credit, leaving $2,739.14." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model incorrectly treated the federal American Opportunity Credit as a nonrefundable Oklahoma tax credit. The applicable state reduction is the $220 Oklahoma child care/child tax credit derived from the $4,400 federal CTC." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Although the model identified $75,303 as the taxable-income figure and the correct $220 credit, it computed regular Oklahoma tax as $3,064 instead of $2,959.14. It therefore applied the wrong 2026 joint-filer rate schedule or bracket thresholds." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model invoked an overtime deduction but still produced $4,223.29, which is inconsistent with the traced Oklahoma deduction and bracket sequence. It failed to use the $22,429.78 itemized deduction and the $220 nonrefundable child credit that produce $2,739.14." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model limited itemized deductions to $17,000 and therefore used taxable income of $71,003 instead of the traced Oklahoma itemized-deduction computation. Even after correctly estimating a $220 credit, the wrong deduction base left its liability $58.86 too high." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used a flat 4.5% approximation instead of Oklahoma's graduated 2026 joint-filer brackets. It also substituted two $100 child credits for the single $220 Oklahoma credit derived as 5% of the $4,400 federal CTC." +us,scenario_046,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The unexplained $3,800 estimate does not apply the traced $22,429.78 itemized deduction, Oklahoma bracket tax of $2,959.14, and $220 nonrefundable credit. Those steps yield $2,739.14, not a rounded gross-tax estimate." +us,scenario_046,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model deducted $26,309 of federal-style itemized expenses rather than the traced Oklahoma itemized amount of $22,429.78. It also stopped at bracket tax and omitted the $220 nonrefundable Oklahoma child credit." +us,scenario_046,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model invented taxable income of $45,216 and applied a flat 4.5% rate. Oklahoma uses the traced deduction and exemption calculation, graduated joint-filer brackets producing $2,959.14, and a $220 nonrefundable credit." +us,scenario_046,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the starting AGI to about $77,164 and then applied an approximate $15,000 standard deduction instead of beginning with $92,003 and using the traced $22,429.78 itemized deduction. It also stated that no state credit applied, omitting the $220 Oklahoma child credit." +us,scenario_046,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. +us,scenario_046,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the $12,700 standard deduction, omitted all four $1,000 exemptions, and therefore taxed $79,303 rather than following the traced deduction calculation. It also incorrectly concluded that no Oklahoma nonrefundable credit applied, omitting the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The unexplained $3,773 answer does not reproduce the traced Oklahoma calculation. The applicable bracket tax is $2,959.14 and the $220 nonrefundable child credit reduces it to $2,739.14." +us,scenario_046,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model overstated federal AGI as approximately $94,004 despite the listed $92,000 of wages and $3 of interest, then used the $12,700 standard deduction and omitted four exemptions. It also failed to subtract the $220 Oklahoma nonrefundable child credit." +us,scenario_046,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model used obsolete or incorrect Oklahoma brackets ranging from 0.5% to 5% rather than the applicable 2026 schedule. It also asserted that no nonrefundable credit applied, omitting the $220 Oklahoma child credit derived from the $4,400 federal CTC." us,scenario_046,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_046,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_048,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented Ohio aged Medicaid eligibility from age 85 and assets below $2,000 without applying PolicyEngine's category gate. Under the trace, the person has medicaid_category NONE, 2.17 FPL MAGI income, and no SSI receipt, so low bank assets do not create Medicaid eligibility." -us,scenario_048,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated SSI-related aged Medicaid disregards and Ohio expansion as producing an aged Medicaid category for this person. PolicyEngine did not assign any aged, SSI-related, or expansion Medicaid category: MAGI income is 2.17 FPL and the category result is NONE." -us,scenario_048,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model assumed PolicyEngine has an Ohio aged pathway with higher effective thresholds that this 85-year-old with $1,000 in assets satisfies. The engine trace assigns no qualifying Medicaid category, so age and modest assets do not override the failed MAGI-category result at 2.17 FPL." -us,scenario_048,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model misapplied SSI-related aged Medicaid and Medicare Savings Program reasoning, including the false premise that most Social Security is excluded for this eligibility result. PolicyEngine's Medicaid eligibility output requires a qualifying Medicaid category, and this person receives no SSI and is assigned medicaid_category NONE." -us,scenario_048,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model collapsed the case to age 85 plus 'very low income' and skipped the Medicaid category determination. PolicyEngine computes MAGI income at 2.17 FPL and assigns medicaid_category NONE, so the person is not eligible despite age and Ohio residence." -us,scenario_048,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's stated derivation computes payroll tax from covered wages only and lands near the reference, but it submitted 782 instead of the amount its own explanation supports. The submitted value is an unexplained final-answer mismatch, not a substantive payroll-tax rule calculation." -us,scenario_048,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly computed that the ordinary allotment formula produces $0, but then awarded the $23 minimum allotment without first establishing income eligibility. It also submitted the monthly minimum of $23 as the annual output after explicitly calculating $276 annually." -us,scenario_048,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model explicitly found net income above its stated eligibility limit and concluded that the household was ineligible, yet submitted $2,376. The minimum benefit cannot override a failed income test, and $2,376 is unsupported by the model’s own computation." -us,scenario_048,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated elderly status as sufficient to bypass the controlling income threshold and assumed a minimum allotment. With annual income of $34,626 against the $17,420 one-person threshold, the household is ineligible and receives no minimum benefit." -us,scenario_048,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model asserted that medical and shelter deductions reduce net income to roughly $0–$100 per month, but the listed medical costs produce only a small excess-medical deduction and $650 monthly rent is below half of income after the preceding deductions, producing no excess-shelter deduction. It therefore invented near-zero net income and a maximum allotment instead of applying the income threshold that yields $0." -us,scenario_048,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model labeled $34,626 of annual wages and Social Security as “very low countable income” and declared the household eligible without applying the one-person SNAP income threshold. That income exceeds the $17,420 threshold, so the estimated $225 monthly benefit has no eligibility basis." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model included $721 too much in AGI, using $247,385 instead of $246,663.72, and therefore obtained taxable income of $190,185 instead of $188,883.95. It then submitted $33,223 despite its own recomputation yielding $30,830, so the final value does not follow its stated calculation." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $25,000 qualified-overtime deduction and used a $29,200 standard deduction instead of $32,200. It also treated the nondeductible $721 traditional IRA contribution as deductible and applied the ordinary rate schedule without correctly separating the $6,208 of qualified dividends." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the $25,000 qualified-overtime deduction, leaving taxable income at $215,885 instead of $188,883.95. It also inserted $27 of Additional Medicare Tax into an income-tax output that excludes payroll tax, then submitted $49,861 despite calculating only about $36,571." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted the $25,000 qualified-overtime deduction and used an understated standard deduction, producing taxable income of $217,085 instead of $188,883.95. Its submitted $53,503 also contradicts its own stated total of about $37,055." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model incorrectly totaled itemized deductions at about $46,081 even though the listed components sum to $26,736, while the $32,200 standard deduction is larger. It also omitted the $25,000 overtime deduction and applied tax rates that cannot produce $47,500 from its stated taxable income." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the $25,000 qualified-overtime deduction and used a $30,000 standard deduction rather than $32,200. Those errors left taxable income at $217,385 rather than $188,883.95 and overstated ordinary income subject to the rate schedule." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $25,000 qualified-overtime deduction and wrongly deducted the $721 traditional IRA contribution despite recognizing the phaseout. It then added unsupported amounts for Medicare tax and phaseouts and submitted $62,700 even though its own income-tax computation was $38,831." -us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the $16,408 employer-sponsored insurance premium a second time even though the trace treats gross wages as reduced only by the traditional 401(k) contribution. It also used expired pre-TCJA brackets and personal-exemption concepts, itemized $26,736 instead of taking the $32,200 standard deduction, and omitted the $25,000 overtime deduction." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model double-counted the $16,408 employer health-premium exclusion, used itemized deductions and personal exemptions under an expired-law framework, and omitted the $25,000 overtime deduction. The correct deduction path is the $32,200 standard deduction plus the overtime deduction from AGI of $246,663.72." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies taxable income of about $229,000, but the trace yields $188,883.95 after the $32,200 standard deduction and $25,000 overtime deduction. It therefore failed to apply the controlling deductions and taxed roughly $40,000 too much income." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model double-counted the employer health-premium exclusion and substituted pre-TCJA itemized deductions and personal exemptions for the applicable deduction structure. It also omitted the $25,000 qualified-overtime deduction that reduces taxable income to $188,883.95." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied an expired-law combination of itemized deductions and personal exemptions and omitted the $25,000 qualified-overtime deduction. This produced taxable income of $194,141 under the wrong tax regime instead of the traced $188,883.95." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model double-counted the $16,408 employer health-premium exclusion, reducing AGI to $230,977, and omitted the $25,000 overtime deduction. It also used a $30,000 standard deduction instead of $32,200, so its $200,977 taxable-income figure does not follow the applicable deduction rules." -us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model gave no computation supporting $20,460 and referred generically to itemized deductions and a potentially deductible IRA contribution. The trace instead uses the $32,200 standard deduction, no IRA deduction, and the $25,000 overtime deduction to reach $188,883.95 of taxable income and $30,543.91 of tax." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model supplied no deduction or rate calculation supporting $38,566. The correct computation taxes $188,883.95 after the $32,200 standard deduction and $25,000 overtime deduction, with $6,208 separated for preferential qualified-dividend treatment." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $16,408 employer health-premium exclusion and incorrectly added a separate $580 charitable deduction on top of the standard deduction. These errors reduced taxable income to $173,197 instead of $188,883.95." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model improperly stacked a $580 charitable deduction and a vehicle-loan-interest deduction on top of the $32,200 standard deduction and $25,000 overtime deduction. The trace allows total deductions of $57,779.77 and yields $30,543.91, not $30,498.38." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model double-counted the $16,408 employer health-premium exclusion and added a $580 cash-charity deduction alongside the standard deduction. That drove taxable income down to $173,197 rather than the traced $188,883.95." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model applied a vehicle-loan-interest deduction that is absent from the traced $57,779.77 deduction total. Its submitted tax also fails to follow the traced taxable income of $188,883.95 and the separate $931.20 tax on qualified dividends." -us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used roughly $263,000 of AGI and about $230,000 of taxable income, failing to subtract the $15,436 traditional 401(k) contribution, the $32,200 standard deduction, and the $25,000 overtime deduction correctly. The trace yields AGI of $246,663.72 and taxable income of $188,883.95." -us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used itemized deductions plus personal exemptions under an expired-law framework and omitted the $25,000 qualified-overtime deduction. The applicable computation instead takes the $32,200 standard deduction and produces taxable income of $188,883.95." -us,scenario_049,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model grounded its answer in itemized mortgage, property-tax, and charitable deductions even though the $32,200 standard deduction controls. It also omitted the $25,000 qualified-overtime deduction, leaving too much income subject to tax." -us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model double-counted the $16,408 employer health-premium exclusion, then used itemized deductions and personal exemptions under an expired-law framework. It also omitted the $25,000 qualified-overtime deduction, whereas the trace uses the $32,200 standard deduction plus that overtime deduction." -us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly stacked a $580 nonitemizer charitable deduction on top of the $32,200 standard deduction. It also started from AGI of $247,385 rather than $246,663.72, producing taxable income of $189,605 instead of $188,883.95." -us,scenario_049,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the $25,000 overtime deduction as an AGI adjustment, additionally deducted the nondeductible $721 traditional IRA contribution, and used $25,675 rather than the $25,000 overtime cap. It then used a $30,900 standard deduction instead of $32,200 and applied inaccurate 2026 brackets." -us,scenario_049,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model arithmetically misapplied the $15,436 401(k) subtraction, reporting $258,821 instead of $247,385 before the trace's additional adjustment, and claimed $15,395 of itemized deductions exceeded a $32,000 standard deduction. It omitted the $25,000 overtime deduction, treated all $6,320 of dividends as qualified, and submitted $51,214.75 despite its own recomputed regular tax of $42,933.38." +us,scenario_048,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated age and assets below $2,000 as sufficient for Ohio aged Medicaid while never establishing that the head passed the applicable income test. The head's wages and Social Security total $34,626, SSI is zero, and no Medicaid category applies." +us,scenario_048,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model asserted that SSI-related disregards reduce roughly $2,886 of monthly wages and Social Security to well below the aged Medicaid income limit, but the cited $20 general disregard and earned-income disregard do not produce that result. It also incorrectly invoked Medicaid expansion for an aged applicant, leaving the head with no qualifying category." +us,scenario_048,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model assumed Ohio's aged Medicaid pathway has an unspecified higher effective income threshold and treated low assets as enough to qualify. It never applied a threshold that admits the head's $34,626 annual wages and Social Security, and the head qualifies through no aged or MAGI category." +us,scenario_048,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly claimed that most Social Security survivor benefits are excluded under SSI-related Medicaid counting and conflated Medicare Savings Program eligibility with regular Medicaid eligibility. Age, low assets, and possible QMB, SLMB, or QI status do not establish eligibility for the requested Medicaid output, and no Medicaid category applies." +us,scenario_048,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model labeled the head's income ""very low"" and inferred Medicaid eligibility directly from age and residence. The head's MAGI income is 2.17 times FPL, SSI is zero, and age 85 alone does not supply a qualifying Medicaid pathway." +us,scenario_048,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted eligibility based on age and income without identifying or applying any Ohio Medicaid pathway. The head's age does not independently confer eligibility, and the income facts result in no qualifying Medicaid category." +us,scenario_048,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model identified the correct payroll-tax base and components but miscalculated their amounts, rounded the annual liability to a whole dollar, and then submitted $782 despite its own stated total of $586. Applying 6.2% and 1.45% to $7,656 yields $474.67 and $111.01, respectively, for $585.68." +us,scenario_048,snap,claude-opus-4.7,llm_error,thresholds_rates,False,The model correctly calculated that the benefit formula produced $0 but then applied the $23 minimum allotment to an income-ineligible household. It also submitted $23 as an annual amount after stating that $23 was monthly and that twelve months would equal $276. +us,scenario_048,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model explicitly found that net income exceeded its stated eligibility limit and that the household was ineligible, yet submitted $2,376. It applied a positive allotment after its own eligibility calculation had already required a $0 benefit." +us,scenario_048,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model treated age 85 as bypassing the controlling income screen and assumed the minimum allotment remained available. The household’s $34,626 income exceeds the $17,420 eligibility threshold, so no minimum benefit is payable." +us,scenario_048,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model asserted that medical and shelter deductions reduced net income to roughly $0–$100 without performing the arithmetic. The listed medical expenses yield only a small excess-medical deduction, and $650 monthly rent is below half of the roughly $2,519 monthly income after the deductions it identified, so there is no excess-shelter deduction and no near-maximum allotment." +us,scenario_048,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled $34,626 of annual wages and Social Security as “very low countable income” and assigned an estimated $225 monthly benefit without applying the SNAP income threshold. That income exceeds the $17,420 limit for a one-person household, producing $0." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It denied the traditional IRA deduction, leaving AGI and taxable income about $721 too high. Its submitted $33,223 also contradicts its own stated recomputation of $30,830 instead of following the calculation through to the required output." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It omitted the separate $25,000 qualified-overtime deduction and used a $29,200 standard deduction instead of $32,200. It also treated the $16,408 employer premium as an additional wage adjustment even though the reference wage computation only subtracts the traditional 401(k), and then applied the rate schedule incorrectly to its own taxable income." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It omitted both the deductible traditional IRA contribution and the $25,000 qualified-overtime deduction, and used an estimated $31,500 standard deduction rather than $32,200. It also incorrectly considered Additional Medicare Tax part of federal income tax and submitted $49,861 despite its own recomputation yielding $36,571." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the deductible traditional IRA contribution and the $25,000 qualified-overtime deduction and understated the standard deduction at $30,300. Its submitted $53,503 bears no relation to its own stated total of about $37,055." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 qualified-overtime deduction and the deductible traditional IRA contribution, while incorrectly choosing itemization over the $32,200 standard deduction. It then substantially overstated the tax on the approximately $195,000 of ordinary income it claimed." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It treated the traditional IRA contribution as nondeductible, omitted the $25,000 qualified-overtime deduction, and used a $30,000 estimated standard deduction instead of $32,200. Those errors left taxable income $28,501 above the traced $188,883.95." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It failed to subtract the $25,000 qualified-overtime deduction and used an estimated $32,600 standard deduction rather than $32,200. It then abandoned its own $38,831 calculation and submitted $62,700 after improperly invoking payroll-tax and unspecified phaseout adjustments." +us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It incorrectly subtracted the $16,408 employer-sponsored insurance premium from the stated gross wages, omitted the $25,000 overtime deduction, and applied obsolete pre-TCJA deductions and rate brackets. The applicable computation uses the $32,200 standard deduction with no personal exemptions." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It double-counted the $16,408 employer premium as a wage exclusion, omitted the $25,000 overtime deduction, and applied itemized deductions plus personal exemptions under an erroneous TCJA-sunset regime. The 2026 computation instead uses the $32,200 standard deduction and no personal exemptions." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its answer implies taxable income of roughly $229,000, but the deductible IRA contribution, $32,200 standard deduction, and $25,000 overtime deduction reduce taxable income to $188,883.95. It therefore failed to apply the required deductions before using the tax schedule." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It incorrectly removed the $16,408 employer premium from the stated wages and used itemized deductions plus personal exemptions under a pre-TCJA framework. It also omitted the separate $25,000 qualified-overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used itemized deductions and personal exemptions under an erroneous post-TCJA-expiration framework, producing $194,141 of taxable income. The applicable deductions are the $32,200 standard deduction and $25,000 overtime deduction, yielding $188,883.95." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It incorrectly excluded the $16,408 employer premium again from the stated gross wages and omitted the $25,000 qualified-overtime deduction. It also used a $30,000 standard deduction instead of $32,200." +us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,It supplied no numeric output or explanation for the requested variable. +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"Its $20,460 answer is incompatible with the traced $188,883.95 taxable income and the 2026 joint rate schedule, which produces $29,612.71 before adding qualified-dividend tax. Its generic explanation identifies no calculation capable of supporting the submitted amount." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Its $38,566 answer overstates the tax generated by $188,883.95 of taxable income and $6,208 of qualified dividends. The generic explanation never applies the $25,000 overtime deduction or shows the ordinary-bracket calculation." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It incorrectly excluded the $16,408 employer health premium from the stated wage amount and added a $580 nonitemizer charitable deduction. Those errors reduced taxable income to $173,197 instead of $188,883.95." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It improperly stacked a cash-charity deduction and vehicle-loan-interest deduction on top of the standard deduction. The traced deduction total consists principally of the $32,200 standard deduction and $25,000 overtime deduction, with taxable income of $188,883.95." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It incorrectly excluded pretax health premiums from the already stated wage total and added a $580 cash-charity deduction alongside the standard deduction. This understated taxable income by $15,686.95." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It treated vehicle-loan interest as an allowable deduction in addition to the standard and overtime deductions. The correct taxable-income computation uses AGI of $246,663.72 and total deductions of $57,779.77, after which the tax is $30,543.91." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It used roughly $263,000 of AGI and $230,000 of taxable income, failing to subtract the traditional 401(k), deductible IRA contribution, and $25,000 overtime deduction correctly. The traced taxable income is $188,883.95." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used itemized deductions and personal exemptions under an erroneous TCJA-sunset treatment and omitted the $25,000 qualified-overtime deduction. The household instead uses the $32,200 standard deduction and has no personal-exemption subtraction." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its explanation relies on itemized mortgage, property-tax, and charitable deductions and never applies the $25,000 qualified-overtime deduction. The household uses the $32,200 standard deduction, producing taxable income of $188,883.95." +us,scenario_049,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It started from AGI of $230,256 by improperly excluding employer health premiums from the stated wages, then omitted the $25,000 overtime deduction. These offsetting mistakes produced taxable income of $198,056 rather than $188,883.95." +us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It incorrectly excluded the $16,408 employer premium from stated gross wages and applied itemized deductions plus personal exemptions under a pre-TCJA framework. It also omitted the $25,000 qualified-overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It failed to deduct the traditional IRA contribution and improperly added a $580 nonitemizer charitable deduction. Those errors netted to taxable income of $189,605 rather than $188,883.95." +us,scenario_049,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It treated the overtime deduction as $25,675 and subtracted it in deriving AGI rather than applying the capped $25,000 deduction after AGI. It also used a $30,900 estimated standard deduction instead of $32,200 and misapplied the ordinary brackets." +us,scenario_049,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It made an arithmetic error by subtracting $15,436 from $262,821 as though the result were $258,821, then incorrectly claimed $15,395 of itemized deductions exceeded a $32,000 standard deduction. It omitted the $25,000 overtime deduction, included all dividends rather than only $6,208 as qualified, and submitted $51,214.75 despite its own recomputation of $42,933.38." +us,scenario_049,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It omitted the $25,000 qualified-overtime deduction, used incorrect retirement and standard-deduction amounts, and applied the joint tax brackets incorrectly. Even its stated $205,441 of ordinary taxable income does not generate $44,586.80 under the applicable schedule." us,scenario_049,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_049,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model did not provide a parseable value for head_medicare_eligible. The required output was 0 because the head is 53 and has no listed Medicare early-eligibility condition. -us,scenario_049,head_medicare_eligible,gpt-5.4-nano,llm_error,health_coverage,False,"The model's reasoning correctly applied the Medicare screen: age 53, not 65+, and no disability or ESRD indicator means the head is not Medicare eligible. It then violated the answer contract by submitting value = 1 while its own explanation says to submit 0." +us,scenario_049,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable output for head_medicare_eligible, violating the required submission contract." +us,scenario_049,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model's reasoning correctly determined that the 53-year-old head lacks both the age threshold and any qualifying early-Medicare condition, but it submitted value 1 instead of 0. Its numeric output directly contradicts its own eligibility analysis and required final marker." us,scenario_049,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_049,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly derived all three components and their $15,134.50 sum, then submitted $14,503.98 instead. Its numeric output contradicts its completed arithmetic." -us,scenario_049,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model used the obsolete $168,600 Social Security wage base and applied the $200,000 Additional Medicare threshold instead of the $250,000 joint threshold. It then added an invented $904.13 adjustment despite correctly stating that New Hampshire has no employee state payroll tax, and even those stated components do not sum to $20,502.83." -us,scenario_049,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model capped Social Security wages at $176,100 instead of the 2026 $184,500 wage base. This understated employee Social Security tax by $520.80 while its Medicare components were correct." -us,scenario_049,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model correctly calculated $11,439 of Social Security tax, $3,668.50 of Medicare tax, and $27 of Additional Medicare Tax, totaling $15,134.50. It then submitted the unrelated value $13,164.85." -us,scenario_049,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model ultimately substituted the 2025 Social Security wage base of $176,100 for the 2026 base of $184,500. That reduced Social Security tax from $11,439 to $10,918.20 and understated the total by $520.80." -us,scenario_049,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a $176,100 Social Security wage base instead of $184,500. Its Medicare tax and $27 Additional Medicare Tax were correct, leaving the entire $520.80 error in employee Social Security tax." -us,scenario_049,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model deducted the listed $16,408 employer-sponsored insurance premiums from FICA wages without an input establishing that payroll exclusion. It also used a $176,100 Social Security cap and eliminated the $27 Additional Medicare Tax by comparing the improperly reduced wage base with the joint threshold." -us,scenario_049,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly reduced Medicare wages from $253,000 to $236,592 by subtracting the listed insurance premiums, eliminating the $27 Additional Medicare Tax. It also used a $183,900 Social Security cap instead of $184,500." -us,scenario_049,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The stated method with a $176,100 Social Security cap, 1.45% Medicare tax on $253,000, and 0.9% on $3,000 yields $14,613.70, not $13,959.81. The model both used the wrong Social Security wage base and submitted a number inconsistent with its own described calculation." -us,scenario_049,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly deducted $16,408 of insurance premiums from FICA wages and omitted the Additional Medicare Tax. It also failed to use the $184,500 Social Security wage base needed to produce the $11,439 Social Security component." -us,scenario_049,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model used an incorrect $174,900 Social Security wage base and improperly reduced Medicare wages to $236,592 for insurance premiums. It consequently omitted the $27 Additional Medicare Tax as well." -us,scenario_049,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly subtracted $16,408 of insurance premiums from Medicare wages, used a $181,200 Social Security cap instead of $184,500, and omitted the $27 Additional Medicare Tax. These errors reduced every applicable component except the tax rates themselves." -us,scenario_049,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract." -us,scenario_049,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The stated rules yield $11,439 plus $3,668.50 plus $27, or $15,134.50. The submitted $19,648 cannot result from those components and reflects an arithmetic or transcription failure." -us,scenario_049,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model described the correct component structure but submitted $15,302 rather than the $15,134.50 obtained from the $184,500 Social Security cap, full Medicare wages, and $27 surtax. Its estimate used an incorrect indexed cap or threshold amount." -us,scenario_049,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model improperly excluded $16,408 of employer-sponsored insurance premiums from FICA wages, which reduced Medicare tax and erased the $27 Additional Medicare Tax. It also capped Social Security wages at $183,600 rather than $184,500." -us,scenario_049,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"The model correctly computed $11,439 of Social Security tax but improperly reduced Medicare wages by the $16,408 insurance-premium amount. Using full wages produces $3,668.50 of Medicare tax and triggers $27 of Additional Medicare Tax, rather than $3,430.58 and zero." -us,scenario_049,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"The model rounded the Social Security wage base to roughly $180,000 and then rounded the final liability to $14,000 despite listing the applicable Medicare components. Exact application of the $184,500 cap and Medicare thresholds yields $15,134.50." -us,scenario_049,payroll_tax,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly stated that Additional Medicare Tax was not triggered even though joint wages exceed $250,000 by $3,000, producing $27 of tax. Its remaining total also fails to reflect the exact $184,500 Social Security wage base." -us,scenario_049,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"The model improperly reduced Social Security and Medicare wages by $16,408 of insurance premiums, used a $183,000 Social Security cap instead of $184,500, and therefore omitted the $27 Additional Medicare Tax. Payroll tax instead uses the full $253,000 wage input for Medicare and the surtax threshold test." -us,scenario_049,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model explicitly reached the correct $184,500 Social Security base and the correct $15,134.50 total before reverting to obsolete wage bases. It then submitted $19,354.50, a number that matches none of its several displayed calculations." +us,scenario_049,payroll_tax,claude-fable-5,llm_error,other,False,"The model explicitly derived the correct $15,134.50 total but submitted $14,503.98 instead. Its numeric output does not implement its own stated component sum." +us,scenario_049,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model used the obsolete $168,600 Social Security wage base instead of $184,500, applied the $200,000 Additional Medicare threshold instead of the household-adjusted $250,000 threshold, and invented a $904.13 New Hampshire adjustment. Those errors replace the required $11,439 Social Security and $27 Additional Medicare components with unrelated amounts." +us,scenario_049,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model capped Social Security wages at $176,100 instead of the 2026 $184,500 wage base. This understated employee Social Security tax by $520.80 while calculating both Medicare components correctly." +us,scenario_049,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model explicitly calculated all three correct components and their $15,134.50 sum, then submitted $13,164.85. The submitted value is an unexplained output inconsistency rather than the result of its stated computation." +us,scenario_049,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model ultimately substituted the 2025 Social Security wage base of $176,100 for the 2026 $184,500 base. That reduced Social Security tax from $11,439 to $10,918.20 and understated the total by $520.80." +us,scenario_049,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a $176,100 Social Security wage base instead of the 2026 $184,500 base. Its Medicare and Additional Medicare calculations were correct, leaving the entire $520.80 shortfall in employee Social Security tax." +us,scenario_049,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the listed $16,408 employer-sponsored insurance premium from FICA wages and also used a $176,100 Social Security cap instead of $184,500. This understated Medicare tax, eliminated the $27 Additional Medicare Tax, and understated Social Security tax." +us,scenario_049,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly reduced Medicare wages from $253,000 to $236,592 by treating the listed employer-sponsored insurance premium as an employee pretax deduction. It also used a $183,900 Social Security cap instead of $184,500 and consequently omitted the $27 Additional Medicare Tax." +us,scenario_049,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The stated formula using a $176,100 Social Security cap, full-wage Medicare tax, and Additional Medicare Tax above $250,000 yields $14,613.70, not $13,959.81. The model both used the wrong Social Security wage base and submitted a number inconsistent with its own described calculation." +us,scenario_049,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly treated the $16,408 employer-sponsored insurance premium as an employee pretax deduction and reduced FICA wages to $236,592. It therefore understated Medicare tax and omitted the $27 Additional Medicare Tax, while its total also fails to identify the required $184,500 Social Security cap." +us,scenario_049,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model used an incorrect $174,900 Social Security wage base and improperly reduced Medicare wages to $236,592 for the listed employer-sponsored insurance premium. It also omitted the $27 Additional Medicare Tax that applies to the $3,000 excess over the household-adjusted threshold." +us,scenario_049,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly reduced Medicare wages to $236,592 by subtracting the listed employer-sponsored insurance premium and used a $181,200 Social Security cap instead of $184,500. This also caused it to omit the $27 Additional Medicare Tax." +us,scenario_049,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so the required output was missing." +us,scenario_049,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The correct components implied by the model's stated categories are $11,439 of Social Security tax, $3,668.50 of Medicare tax, and $27 of Additional Medicare Tax. Its $19,648 answer exceeds that sum by $4,513.50 and therefore applies an unstated extra amount or arithmetic error." +us,scenario_049,payroll_tax,gpt-5.4-nano,llm_error,other,False,"Using the stated categories with the 2026 parameters produces $15,134.50, not $15,302. The model's rounded estimate overstates the exact component sum by $167.50 and does not implement the applicable wage base and threshold precisely." +us,scenario_049,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model improperly subtracted the $16,408 employer-sponsored insurance premium from FICA wages, reducing Medicare wages to $236,592. It also used a $183,600 Social Security cap instead of $184,500 and omitted the $27 Additional Medicare Tax." +us,scenario_049,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"Although it used the correct $11,439 Social Security component, the model improperly reduced Medicare wages by the listed employer-sponsored insurance premium. Full wages produce $3,668.50 of Medicare tax and $27 of Additional Medicare Tax, not $3,430.58 and zero." +us,scenario_049,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"The model rounded the Social Security wage base to about $180,000 and then rounded the resulting total to $14,000 instead of applying the exact $184,500 cap. The exact components it referenced sum to $15,134.50." +us,scenario_049,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly omitted Additional Medicare Tax even though $253,000 exceeds the household-adjusted $250,000 threshold, producing a required $27 surtax. Its remaining $14,808 amount also does not equal Social Security tax at the $184,500 cap plus 1.45% Medicare tax on full wages." +us,scenario_049,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model correctly used the $184,500 Social Security cap but improperly reduced Medicare wages by the $16,408 employer-sponsored insurance premium. Medicare applies to the full $253,000, and the resulting $3,000 excess over the $250,000 threshold also generates $27 of Additional Medicare Tax." +us,scenario_049,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"The model improperly subtracted the listed employer-sponsored insurance premium from both Social Security and Medicare wages and used an estimated $183,000 Social Security cap instead of $184,500. This understated both regular FICA components and incorrectly eliminated the $27 Additional Medicare Tax." +us,scenario_049,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model discussed several obsolete Social Security caps and explicitly computed totals between $14,148.70 and $15,134.50, but then submitted $19,354.50. That submitted number follows none of its stated calculations and exceeds the correct component sum by $4,220." +us,scenario_049,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated components of $11,475, $3,668.50, and $27, which sum to $15,170.50 rather than $23,780.75. It also overstated the Social Security component by using $11,475 instead of the $11,439 generated by the $184,500 wage base." us,scenario_049,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,snap,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,spouse_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_049,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model did not provide a parseable spouse_medicare_eligible value. It therefore failed the output contract rather than applying the Medicare age and disability eligibility rule to the age-52 spouse. -us,scenario_049,spouse_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model's own explanation applied the correct Medicare rule: a 52-year-old spouse with no listed disability or ESRD qualifier is not Medicare eligible. It then submitted value = 1, reversing the binary output that follows from its stated age-based eligibility determination." +us,scenario_049,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no spouse_medicare_eligible output, violating the required output contract." +us,scenario_049,spouse_medicare_eligible,gpt-5.4-nano,llm_error,other,False,"The model's reasoning correctly concluded that the 52-year-old spouse lacked every Medicare eligibility pathway and explicitly stated the flag was 0, but it submitted value = 1. It inverted the binary value at the final output step." us,scenario_049,spouse_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,ssi,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_049,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable output or reasoning for the requested quantity. -us,scenario_049,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model asserted a small positive New Hampshire interest-and-dividends liability without applying the filing requirement threshold to the $5,021 remaining after the joint exemption. That threshold eliminates the liability, and $2,350 does not follow from the stated income, exemption, or applicable tax computation." -us,scenario_049,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model multiplied the $5,021 remaining after the joint exemption by 3% and stopped. It missed the filing requirement threshold, under which that $5,021 produces no New Hampshire liability." -us,scenario_049,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a 1% rate directly to the $5,021 remaining after the joint exemption. It omitted the filing requirement threshold that reduces the liability to zero; it also rounded the resulting $50.21 arithmetic to $50." -us,scenario_049,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model treated the amount remaining after New Hampshire’s exemptions as automatically taxable at 1%. It failed to apply the filing requirement threshold, which prevents any liability on the household’s $5,021 of taxable interest-and-dividend income." -us,scenario_049,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly totaled $9,821 and subtracted the $4,800 joint exemption, but then multiplied $5,021 by 1% without testing the filing requirement threshold. Because $5,021 falls below that threshold, no tax is assessed." +us,scenario_049,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output. +us,scenario_049,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model recognized that New Hampshire taxes only interest and dividends but imposed an unsupported $2,350 liability without applying the filing requirement threshold. The $5,021 remaining after the joint exemption is below that threshold, so the computation stops at $0." +us,scenario_049,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model multiplied the $5,021 remaining after the joint exemption by 3% without first applying New Hampshire's filing requirement threshold. Because $5,021 is below the threshold, no interest-and-dividends tax is assessed." +us,scenario_049,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a 1% rate directly to the $5,021 remaining after the joint exemption and skipped the filing requirement test. That amount is below the filing threshold, producing $0 rather than $50." +us,scenario_049,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model treated the post-exemption interest-and-dividend amount as automatically taxable at 1% and omitted New Hampshire's filing requirement threshold. The $5,021 balance is below that threshold, so the liability is $0." +us,scenario_049,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly totaled $9,821 of interest and dividends and subtracted the $4,800 joint exemption, but then multiplied $5,021 by 1% without testing the filing requirement. Since $5,021 falls below the filing threshold, no tax is assessed." +us,scenario_049,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly derived $5,021 after the joint exemption but treated that balance as subject to the 1% rate automatically. New Hampshire's filing requirement threshold applies before the rate, and the $5,021 balance is below it, yielding $0." us,scenario_049,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model incorrectly added the $180 non-Schedule-D capital-gains input to IRS gross income, producing $24,080 instead of $23,900 of taxable income. Its own bracket calculation then yielded $2,641.60, but it submitted an unsupported $3,086 that does not follow from its reasoning." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model improperly subtracted $5,389 of employer-sponsored insurance premiums from the stated gross wages and then treated the refundable EITC and withholding as eliminating tax before refundable credits. Refundable credits and withholding do not reduce this output; $23,900 of taxable income produces $2,620 before refundable credits." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model incorrectly included the $180 capital-gains input in IRS gross income, but even that shortcut yielded about $2,642 under its stated brackets. It then submitted $2,960 without any computation supporting the additional $318." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model incorrectly added the $180 capital-gains input, giving taxable income of $24,080 rather than $23,900. Its displayed rate calculation produced about $2,642, after which it replaced that result with an unsupported $2,867." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model incorrectly included the $180 capital-gains input in gross income and explicitly calculated about $2,642. It then submitted $2,955, a final value that does not follow from its taxable-income or bracket arithmetic." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $15,000 standard deduction instead of the applicable $16,100 deduction and incorrectly included the $180 capital-gains input in IRS gross income. Those errors produced $25,180 rather than $23,900 of taxable income and therefore overstated the tax." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model incorrectly added the $180 capital-gains input, although its own assumptions still produced only $2,646.60. It then inflated that result to $3,059 through unspecified “adjustments and rounding,” despite no adjustment or rounding capable of creating that difference." -us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a supposed post-TCJA reversion with a personal exemption, a $7,493 standard deduction, and a 15% second bracket. The applicable 2026 rules instead provide a $16,100 single standard deduction and 10%/12% rates over this income range, yielding $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an expired-law structure consisting of an $8,300 standard deduction, a personal exemption, and a 15% second bracket, while also subtracting employer-sponsored premiums from gross wages. The applicable computation uses $40,000 of gross income, a $16,100 standard deduction, and the 10% and 12% brackets." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model invented a $1,300 nonrefundable child tax credit or adjustment for a single taxpayer with no child or other dependent. No nonrefundable credit applies, and the tax on $23,900 of taxable income remains $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model subtracted the employer-sponsored insurance amount from gross wages and used a reverted-law combination of a smaller standard deduction and personal exemption. The applicable calculation starts from $40,000, subtracts the $16,100 standard deduction, and taxes $23,900 under the 2026 10% and 12% brackets." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model double-counted the employer-sponsored insurance premiums as a reduction from stated gross wages and applied an obsolete personal exemption, smaller standard deduction, and 15% bracket. Current 2026 treatment instead taxes $23,900 after the $16,100 standard deduction at 10% and 12%." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the stated $40,000 of gross wages by $5,389 of employer-sponsored insurance premiums and used a $15,700 standard deduction. The computation instead uses $40,000 of IRS gross income and a $16,100 deduction, leaving $23,900 taxable." -us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model used a $14,600 standard deduction and incorrectly added the $180 capital-gains input, causing its displayed calculation to produce $2,831.10. It then submitted $2,530 without identifying any credit or arithmetic operation that reduces its calculated amount to that value." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction as eliminating all federal taxable income even though $40,000 minus $16,100 leaves $23,900. With no nonrefundable credits, the 2026 brackets produce $2,620 rather than zero." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model vaguely invoked unspecified “nonrefundable taxes reduced for the modeled situation” and submitted $2,146 without naming a valid deduction or credit. The specified inputs leave $23,900 taxable, and no nonrefundable credit reduces the resulting $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,350 standard deduction instead of $16,100 and incorrectly included the $180 capital-gains input, overstating taxable income by $930. Applying the correct deduction to $40,000 leaves $23,900 and produces $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $180 non-Schedule-D capital-gains input to IRS gross income. That raised taxable income from $23,900 to $24,080 and tax from $2,620 to $2,641.60." -us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used approximately $24,680 of taxable income instead of the traced $23,900. The correct $40,000 gross-income base less the $16,100 standard deduction produces $2,620 under the 2026 brackets." -us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used the 2025 $15,000 standard deduction as a proxy instead of the applicable 2026 amount of $16,100 and also added the $180 capital-gains input. It consequently taxed $25,180 rather than $23,900." -us,scenario_051,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model subtracted employer-sponsored premiums from gross wages and applied a reverted-law personal exemption, reduced standard deduction, and 15% second bracket. The applicable 2026 computation uses $40,000 of gross income, a $16,100 standard deduction, and the 10% and 12% brackets." -us,scenario_051,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not be evaluated as a substantive tax calculation." -us,scenario_051,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $180 capital-gains input and then subtracted roughly $545 as a supposed “0% LTCG adjustment.” This input does not create a nonrefundable credit or justify reducing ordinary bracket tax; the traced taxable income is $23,900 and the tax is $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's stated assumptions produced $2,618.60, but it submitted $1,410 without any deduction, credit, or arithmetic supporting that reduction. The applicable $16,100 deduction and $40,000 gross-income base yield $2,620." -us,scenario_051,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model treated the household as receiving a positive childless EITC despite $40,000 of earnings. For a single filer with no qualifying children, the EITC is fully phased out at this income level, so the EITC component of federal refundable credits is $0." -us,scenario_051,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable value or explanation for federal_refundable_credits. This is a missing-output contract failure rather than a substantive tax calculation. -us,scenario_051,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's reasoning correctly reduced the childless EITC to $0 and stated that no other refundable credits apply, but it submitted $2,557.60 anyway. Its final value contradicts its own computation; the correct sum of the refundable credit components it identified is $0." -us,scenario_051,head_medicaid_eligible,gpt-5.4-nano,llm_error,health_coverage,False,"The model treated Medicaid eligibility as a default for a low-to-moderate wage adult unless a disqualifying status was listed. It failed to apply Louisiana's adult Medicaid income pathway: the head's MAGI is 2.51 times FPL, above the MAGI eligibility limit, and the person has no SSI or other category that would create eligibility." -us,scenario_051,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the $5,389 employer-sponsored insurance premium as a pretax employee deduction from FICA wages. FICA applies to the full $40,000 wage input, yielding $3,060 rather than $2,647.74." -us,scenario_051,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages by the $5,389 ESI premium even though the facts do not identify it as a pretax employee payroll contribution. Applying 7.65% to the full $40,000 produces $3,060." -us,scenario_051,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model misclassified the employer-sponsored insurance premium as a pretax deduction from the employee's FICA wage base. Social Security and Medicare taxes are calculated on all $40,000 of wages, totaling $3,060." -us,scenario_051,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model subtracted the $5,389 insurance premium from wages without an input establishing a pretax payroll deduction. The correct FICA base is $40,000, so the employee payroll tax is $3,060." -us,scenario_051,payroll_tax,glm-5.2,llm_error,other,False,"The model's stated components and arithmetic correctly total $3,060, but it submitted $3,063.77. The submitted value contradicts its own derivation and includes an unexplained $3.77." -us,scenario_051,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly calculated exact employee FICA of $3,060 and then improperly rounded it to $3,100. The requested annual amount is the computed liability, not a two-significant-digit estimate." -us,scenario_051,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model mislabeled the $2,480 Social Security calculation as $3,060, then added the $580 Medicare tax to that erroneous component. At 6.2% of $40,000, Social Security tax is $2,480, and total FICA is $3,060." -us,scenario_051,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $5,389 employer-sponsored insurance premium from FICA wages. Using the full $40,000 wage base gives $2,480 of Social Security tax plus $580 of Medicare tax, totaling $3,060." -us,scenario_051,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax output or explanation. It therefore failed the required structured-output contract rather than performing a substantive calculation. -us,scenario_051,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly derived $2,480 of Social Security tax and $580 of Medicare tax, totaling $3,060, but submitted $3,074.95. The extra $14.95 has no basis in its stated computation." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model incorrectly added the $180 input to the traced $40,000 of IRS gross income, which raises taxable income from $23,900 to $24,080 and produces $2,641.60 rather than $2,620. It then submitted $3,086, contradicting even its own stated $2,641.60 calculation." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model improperly reduced wages by the separately listed employer-sponsored insurance premiums and used the wrong standard deduction, then erased the remaining income tax by applying EITC and withholding concepts. EITC is refundable and withholding is a payment, so neither reduces federal income tax before refundable credits." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model incorrectly added the $180 input to IRS gross income, but its own resulting bracket calculation was approximately $2,642. It then submitted $2,960 without any tax computation supporting the extra $318." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model incorrectly included the $180 input in gross income, leading its written calculation to $2,642 instead of the traced $2,620. It then submitted $2,867 despite explicitly computing $2,642 and identifying no additional tax." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model incorrectly added $180 to the $40,000 gross-income base, but even that error yields about $2,642 under the brackets it stated. Its $2,955 submission does not follow from its taxable income, rates, or assertion that no credits apply." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $15,000 standard deduction instead of $16,100 and included the $180 input, producing $25,180 rather than $23,900 of taxable income. Its use of approximate 2025 thresholds instead of the applicable 2026 parameters further inflated the tax." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model incorrectly included the $180 input, although its own figures still produce only $2,646.60. It then submitted $3,059 based on unspecified “adjustments and rounding,” neither of which can add $412.40 to the stated bracket calculation." +us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a reverted pre-TCJA regime with a small standard deduction, a personal exemption, and a 15% second bracket instead of the applicable 2026 $16,100 standard deduction and 12% second bracket. It also wrongly subtracted the separately listed employer-sponsored insurance premiums from the $40,000 wage amount." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a post-expiration regime with an $8,300 standard deduction, a $5,150 personal exemption, and a 15% second bracket rather than the applicable 2026 $16,100 standard deduction and 12% bracket. It also wrongly deducted $5,389 of employer-sponsored insurance premiums from wages." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model invented a $1,300 nonrefundable child or other credit for a single filer with no dependents and no facts supporting any such credit. It also used the wrong $15,000 standard deduction and included the $180 input in AGI." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced the stated $40,000 wages by $5,389 of employer-sponsored insurance premiums and substituted a combined standard-deduction-and-personal-exemption amount of roughly $13,400. The traced calculation instead subtracts only the $16,100 standard deduction from $40,000." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied an obsolete-expiration structure with a personal exemption and a 15% second bracket instead of the applicable $16,100 standard deduction and 12% bracket. It also improperly subtracted the $5,389 employer-sponsored insurance amount from the stated wages." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $5,389 of employer-sponsored insurance premiums from the stated wages and used a $15,700 standard deduction rather than $16,100. Those errors reduced taxable income to $19,091 instead of the traced $23,900." +us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model used a $14,600 standard deduction rather than $16,100 and included the $180 input, so its own tax calculation was $2,831.10. It then submitted $2,530 with no stated credit or computation reconciling that number; its ACA premium-tax-credit discussion is irrelevant to this output." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction as eliminating all income tax, even though $40,000 minus the $16,100 deduction leaves $23,900 taxable. Applying the 2026 brackets to that amount yields $2,620, and no nonrefundable credit offsets it." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model supplied no concrete deduction, bracket calculation, or identified nonrefundable credit supporting $2,146. The traced $40,000 gross income less the $16,100 standard deduction leaves $23,900 taxable and produces $2,620 with no credit reduction." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,350 standard deduction instead of $16,100 and included the $180 input, overstating taxable income by $930. It also used an approximate $12,000 first-bracket threshold rather than the applicable 2026 threshold." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $180 non-Schedule-D input to the traced federal gross-income base. That produced $24,080 of taxable income and $2,641.60, whereas the traced $23,900 taxable income yields $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used approximately $24,680 of taxable income instead of $23,900 and provided no deduction derivation supporting the extra $780. The applicable calculation is $40,000 minus the $16,100 standard deduction, followed by the 2026 brackets." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used the 2025 $15,000 standard deduction rather than the 2026 $16,100 amount and incorrectly included the $180 input. This overstated taxable income as $25,180 instead of $23,900 and consequently overstated tax." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used reverted pre-TCJA deductions, a personal exemption, and a 15% second bracket instead of the applicable 2026 $16,100 standard deduction and 12% bracket. It also wrongly treated the $5,389 employer-sponsored insurance amount as a deduction from stated wages." +us,scenario_051,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so its response could not be evaluated as a substantive tax calculation." +us,scenario_051,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model first incorrectly included the $180 input and used the wrong first-bracket threshold, then invented a roughly $545 “0% LTCG adjustment.” Non-Schedule-D capital gains do not create a nonrefundable credit or justify subtracting that amount from ordinary income tax." +us,scenario_051,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model used a $16,300 standard deduction and included the $180 input, but its own stated arithmetic still produced $2,618.60. It then submitted $1,410 with no credit, rate, or computation supporting the $1,208.60 reduction." +us,scenario_051,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model asserted that unspecified nonrefundable credits reduced the liability to zero even though the household has no facts supporting such a credit. The $16,100 standard deduction leaves $23,900 taxable, and the resulting $2,620 remains unreduced before refundable credits." +us,scenario_051,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model awarded $560 of childless EITC without applying the phaseout endpoint for a single filer. At $40,000 of earnings, the childless EITC has fully phased out." +us,scenario_051,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for federal_refundable_credits, violating the required output contract." +us,scenario_051,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own EITC calculation correctly reached $0 and stated that no other refundable credits apply, but it submitted $2,557.60. The submitted value contradicts its completed derivation and does not represent any applicable refundable-credit component." +us,scenario_051,federal_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly awarded $3,758 of EITC to a childless single filer earning $40,000. The childless EITC is fully phased out at that income, and an amount of this size also disregards the much lower maximum credit for filers with no qualifying children." +us,scenario_051,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated age 25, $40,000 in wages, and the absence of a stated disqualifying status as sufficient for Medicaid eligibility without identifying a qualifying category. At 2.51 times FPL, the head fails the applicable MAGI income limits and qualifies through no other pathway, yielding Medicaid category NONE and value 0." +us,scenario_051,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The answer asserted eligibility without applying any Medicaid pathway or income threshold. The head's MAGI is 2.51 times FPL, SSI is zero, and no categorical or state-specific pathway applies, so the required result is value 0." +us,scenario_051,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the $5,389 employer-sponsored insurance premium as a pre-tax employee payroll deduction and reduced FICA wages to $34,611. The listed premium does not reduce the $40,000 payroll-tax base, so 7.65% applies to the full wages." +us,scenario_051,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model assumed the ESI premium was paid through a pre-tax salary reduction and subtracted $5,389 from FICA wages. Payroll tax instead uses the full $40,000 wage amount, yielding $3,060." +us,scenario_051,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly classified the $5,389 ESI premium as a pre-tax deduction from Social Security and Medicare wages. Applying 6.2% and 1.45% to the full $40,000 produces $3,060." +us,scenario_051,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model reduced FICA wages by the listed ESI premium even though no pre-tax employee salary reduction was specified. The correct payroll-tax base is $40,000, not $34,611." +us,scenario_051,payroll_tax,glm-5.2,llm_error,other,False,"The model's own component calculations correctly total $2,480 plus $580 to $3,060, but it submitted $3,063.77. The submitted value contradicts its completed arithmetic by $3.77." +us,scenario_051,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly calculated exact employee payroll tax of $3,060 and then improperly rounded it to $3,100. The requested annual amount is the computed liability, not a coarse estimate rounded to the nearest hundred." +us,scenario_051,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model mislabeled $3,060 as the Social Security tax instead of applying the 6.2% rate, which gives $2,480. Adding the correctly calculated $580 Medicare tax to $2,480 yields $3,060, not $3,640." +us,scenario_051,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model subtracted the $5,389 ESI premium from wages without a stated pre-tax salary-reduction arrangement. Social Security and Medicare taxes apply to the full $40,000, producing $3,060 rather than the rounded $2,648." +us,scenario_051,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax value or explanation. It therefore failed the required structured-output contract. +us,scenario_051,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly derived $2,480 of Social Security tax plus $580 of Medicare tax and explicitly totaled them as $3,060, but submitted $3,074.95. The submitted value is inconsistent with every component in its explanation." us,scenario_051,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model first used the wrong $40,180 AGI and $12,500 deduction to calculate $830.40, then submitted $1,020 with no computation supporting that value. The required computation is 3% of $40,000 minus $12,835, which equals $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model falsely treated Louisiana as a state without an individual income tax. Louisiana imposes a 3% tax on the $27,165 taxable amount, producing $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model included the separate $180 capital-gains input in Louisiana AGI and used a $12,500 standard deduction. The engine uses $40,000 of AGI and a $12,835 deduction, leaving $27,165 taxable and $814.95 of tax." -us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly multiplied its own asserted $27,680 base by 3% to get $830.40, then invented an unspecified personal credit and submitted $632. No such reduction applies; the correct $27,165 base produces $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model used the wrong $40,180 income base and $12,500 deduction, then replaced its calculated tax of about $830 with $812 through an unsupported rounding adjustment. Ordinary rounding cannot create that reduction; 3% of the engine’s $27,165 taxable income is $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted estimated federal income tax from Louisiana income and used a $12,500 state deduction, while also including the $180 input in AGI. The applicable derivation uses $40,000 of AGI less the $12,835 Louisiana standard deduction, with no federal-tax deduction in this calculation, yielding $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model calculated $830.40 from its asserted base but then submitted $1,080, even though its claimed additional exemption would reduce rather than increase tax. The correct base is $27,165 and the 3% liability is $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied an obsolete 4.25% rate, used an unexplained $34,791 AGI, and substituted a $4,500 exemption for the applicable standard deduction. For 2026 the calculation applies 3% to $40,000 less $12,835, producing $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the former 1.85%/3.5% progressive brackets and deducted federal tax plus a $4,500 exemption from an unexplained $34,791 AGI. The 2026 rule uses a flat 3% rate on $40,000 less the $12,835 standard deduction, yielding $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model explicitly applied Louisiana’s progressive rate schedule instead of the 2026 flat 3% rate. The applicable taxable income is $27,165 after the $12,835 standard deduction, so the liability is $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used an unexplained $34,791 AGI, a $4,500 exemption, and the former 1.85%/3.5% brackets. The correct 2026 computation is 3% of $40,000 minus $12,835, or $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly deducted federal income tax and a $4,500 exemption and then applied the former progressive brackets. Louisiana’s 2026 calculation instead applies 3% to $27,165 after the $12,835 standard deduction, producing $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used an unexplained $34,791 AGI, the old $4,500 exemption, and the former 1.85%/3.5% schedule. The applicable flat-rate computation is 3% of $27,165, yielding $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included the separate $180 capital-gains input in AGI and used a $12,500 deduction. The engine’s AGI is $40,000 and its 2026 Louisiana standard deduction is $12,835, so taxable income is $27,165 and tax is $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and exemptions eliminate the liability without identifying any deduction large enough to do so. The $12,835 standard deduction leaves $27,165 taxable, and the 3% tax is $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model submitted a rough $781 estimate without specifying the adjustments or arithmetic that generate it. The defined computation subtracts the $12,835 standard deduction from $40,000 and applies 3%, yielding $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used $40,180 as AGI and a $12,500 standard deduction, overstating taxable income at $27,680. The engine uses $40,000 less $12,835, so 3% applies to $27,165 and produces $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model included the $180 input in AGI and used a $12,500 single deduction. The correct inputs are $40,000 of AGI and a $12,835 standard deduction, giving $814.95 at 3%." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied 3% to $40,180 less a $12,500 deduction. Louisiana taxable income in the trace is instead $40,000 less $12,835, or $27,165, producing $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used an overstated $27,680 taxable-income base derived from $40,180 and a $12,500 deduction. The applicable base is $27,165 after subtracting $12,835 from $40,000, so the tax is $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model submitted $680 without naming the deductions, credits, rate, or arithmetic behind it. The specified derivation leaves $27,165 taxable after the standard deduction and applies 3%, producing $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly subtracted the federal standard deduction to determine Louisiana taxable income and then used the former 1.85%/3.5% state brackets. Louisiana uses its own $12,835 standard deduction and a flat 3% rate here, yielding $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model began from an unexplained $34,791 AGI, deducted estimated federal tax and two separate $4,500 allowances, and applied the former progressive brackets. The 2026 computation uses $40,000 less one $12,835 standard deduction and taxes the resulting $27,165 at 3%, producing $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. It therefore failed the required output contract rather than completing the Louisiana tax computation. -us,scenario_051,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model included the separate $180 input in AGI and used a $12,500 deduction. The engine uses $40,000 of AGI less the $12,835 standard deduction, so the 3% liability is $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model contradicted itself about whether Louisiana has an income tax, used an obsolete $4,500 deduction to calculate $1,065, and then submitted an unsupported $1,208. The applicable calculation is 3% of $40,000 less $12,835, which equals $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model subtracted an estimated federal standard deduction and an obsolete $4,500 Louisiana exemption, then applied an incorrect progressive rate schedule. Louisiana’s 2026 calculation uses a single $12,835 state standard deduction and a flat 3% rate, yielding $814.95; its submitted $573 also does not match its own $482.52 calculation." +us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model first included the $180 capital gain and used a $12,500 deduction, producing $830.40 instead of using $40,000 minus $12,835. It then submitted $1,020, contradicting its own arithmetic without any additional computation." +us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model falsely treated Louisiana as a state without an individual income tax. Louisiana applies a 3% tax to the $27,165 taxable income in this case." +us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model included the $180 capital gain in Louisiana AGI and used a $12,500 standard deduction. The traced inputs use $40,000 of AGI and a $12,835 deduction, leaving $27,165 rather than $27,680." +us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model computed $830.40 from an incorrect $40,180 AGI and $12,500 deduction, then invented an unspecified personal credit to submit $632. No such reduction applies before refundable credits here." +us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used $40,180 of income and a $12,500 deduction instead of the traced $40,000 AGI and $12,835 deduction. It then replaced its own $830 calculation with an unsupported rounding adjustment to $812." +us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly deducted estimated federal income tax and used a $12,500 Louisiana deduction against $40,180. The applicable computation deducts only the $12,835 Louisiana standard deduction from $40,000, producing $27,165 of taxable income." +us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model calculated $830.40 using the wrong AGI and deduction, then submitted $1,080 despite describing an additional deduction that would lower tax. The submitted value has no support in its reasoning or Louisiana computation." +us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied an obsolete 4.25% rate and a $4,500 exemption to an unexplained $34,791 AGI. The applicable calculation uses the 3% rate on $40,000 minus the $12,835 standard deduction." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an unexplained $34,791 AGI, deducted estimated federal tax, and applied the obsolete 1.85%/3.5% schedule with a $4,500 deduction. The applicable method taxes $27,165 at a flat 3%." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies an obsolete progressive-rate calculation rather than the applicable flat-rate computation. Louisiana taxes $27,165 of taxable income at 3%, yielding $814.95." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model substituted an unexplained $34,791 AGI and $4,500 exemption, then applied the obsolete 1.85%/3.5% brackets. The correct base is $40,000 minus $12,835 and the applicable rate is 3%." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model deducted estimated federal income tax and a $4,500 exemption, then used the obsolete 1.85%/3.5% schedule. The applicable calculation uses a $12,835 standard deduction and a flat 3% rate." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used an unexplained $34,791 AGI, a $4,500 deduction, and obsolete progressive brackets. The applicable computation is 3% of $27,165." +us,scenario_051,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included the $180 capital gain in Louisiana AGI and used a $12,500 standard deduction. The trace instead uses $40,000 of AGI and a $12,835 deduction, reducing taxable income to $27,165." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that deductions and exemptions eliminate the Louisiana tax base. The $12,835 standard deduction leaves $27,165 taxable at 3%, and no nonrefundable credit offsets that liability." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model supplied an unsupported estimate without identifying its adjustments. The specified computation leaves $27,165 after the $12,835 standard deduction and taxes it at 3%, producing $814.95 rather than $781." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model included the $180 capital gain in Louisiana AGI and used a $12,500 standard deduction. The traced Louisiana base is $40,000 minus $12,835, or $27,165." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used $40,180 of income and a $12,500 deduction instead of the traced $40,000 AGI and $12,835 standard deduction. This overstated taxable income by $515." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model included the $180 capital gain and understated the standard deduction at $12,500. The applicable taxable income is $27,165, not $27,680." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model based tax on $40,180 less a $12,500 deduction. The trace uses $40,000 less the $12,835 Louisiana standard deduction, yielding $27,165." +us,scenario_051,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model gave an unsupported $680 estimate without specifying the deductions or credits used. The applicable $12,835 deduction leaves $27,165 taxable at 3%, with no stated adjustment that reduces the result to $680." +us,scenario_051,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly subtracted the federal standard deduction to construct Louisiana taxable income and then applied obsolete 1.85%/3.5% brackets. Louisiana instead subtracts its $12,835 standard deduction from $40,000 and applies a flat 3% rate." +us,scenario_051,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an unexplained $34,791 AGI, deducted federal tax plus two $4,500 allowances, and applied obsolete progressive brackets. The applicable base is $27,165 after one $12,835 Louisiana standard deduction, taxed at 3%." +us,scenario_051,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model included the $180 capital gain and used a $12,500 standard deduction rather than the traced $40,000 AGI and $12,835 deduction. It also rounded its already incorrect $830.40 result to a whole dollar." +us,scenario_051,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required output contract." +us,scenario_051,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model included the $180 capital gain in Louisiana AGI and used a $12,500 deduction. The traced computation uses $40,000 minus $12,835, leaving $27,165." +us,scenario_051,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model contradicted itself about whether Louisiana imposes income tax, used an obsolete $4,500 deduction, calculated $1,065, and then submitted $1,208. The applicable calculation is 3% of $40,000 minus $12,835." +us,scenario_051,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used an estimated federal standard deduction, an obsolete $4,500 personal exemption, and obsolete progressive Louisiana rates instead of the $12,835 Louisiana deduction and flat 3% rate. It also submitted $573 despite its own arithmetic producing $482.52." +us,scenario_051,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The submitted $1,324.80 implies an unsupported taxable base far above the traced $27,165. Louisiana applies 3% to $40,000 minus the $12,835 standard deduction, producing $814.95." us,scenario_051,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model's stated calculation reached about $107,277 and then submitted $117,285 with no supporting computation. It also used estimated brackets and a $32,600 deduction instead of the engine's 2026 parameters, rather than computing $103,865.78 of main tax plus $127.20 and $218.42." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model first deducted the entire $29,922 capital loss and then used an obsolete $28,700 standard deduction; its later correction still produced inconsistent taxable-income and tax figures. The applicable computation limits the capital-loss treatment through the traced adjustments, uses the $32,200 standard deduction, and taxes $472,578.56." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used approximate bracket thresholds and a $32,300 standard deduction, then added the $2,250 Additional Medicare Tax even though the requested variable excludes employee payroll taxes. The income-tax output contains main tax, qualified-dividend tax, and NIIT only." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model miscomputed AGI as about $505,748, failing to subtract the $926 pre-tax 401(k) contribution in its displayed arithmetic, and used an estimated $32,300 deduction. It also omitted the $218.42 NIIT from the submitted total." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model arithmetically misapplied the brackets: tax on roughly $471,000 of ordinary taxable income under the stated 10/12/22/24/32 percent schedule is about $104,000, not $141,500. It also failed to add the separately computed $218.42 NIIT coherently." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model included $2,250 of Additional Medicare Tax in federal income tax before refundable credits, although the benchmark assigns it to payroll_tax. It also used an estimated $30,000 standard deduction instead of $32,200; the requested total is main tax plus $127.20 of qualified-dividend tax and $218.42 of NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,period_annualization,False,"The model annualized the explicitly annual $926 401(k) and $43 IRA inputs by multiplying them by 52, contrary to the prompt's full-year convention. Its final $121,500 then abandoned its own calculated $94,754 without a valid tax computation." -us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $16,208 employer-sponsored insurance premium from wages even though the trace treats the listed $500,000 as gross wages reduced only by the $926 pre-tax 401(k) contribution. It also used $8,748 rather than $5,748 as net investment income, failing to apply the $3,000 capital-loss deduction in the NIIT base." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented $45,620 of mortgage-interest and state/local-tax deductions even though only a mortgage balance was listed and unlisted expenses are zero. It also applied a supposed TCJA sunset instead of the enacted 2026 brackets and $32,200 standard deduction." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the full capital losses as reducing AGI to about $476,323 instead of following the traced capital-loss limitation and above-the-line deductions. Its $139,198 result is also inconsistent with tax on the income level it stated." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The submitted $123,753 implies the wrong 2026 bracket schedule or an arithmetic error in applying it. Tax on $472,578.56 is $103,865.78 before adding $127.20 of qualified-dividend tax and $218.42 of NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly assumed expiration of the governing rate and deduction regime, using a $15,300 standard deduction, personal exemptions, and pre-TCJA-style parameters. It also improperly reduced wages by the $16,208 employer premium; the trace uses the $32,200 joint standard deduction and enacted 2026 brackets." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model supplied no computation supporting $113,421, and that number implies misapplication of the 2026 joint brackets. The traced taxable income produces $103,865.78 of main tax, with only $345.62 added for qualified dividends and NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model incorrectly treated TCJA provisions as expired and applied a $15,900 deduction, personal exemptions, and the old 10/15/25/28/33/35 percent schedule. The applicable 2026 computation uses the $32,200 standard deduction and current 10/12/22/24/32-percent brackets at this income." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model severely undercomputed tax on $472,578.56 of joint taxable income. Applying the 2026 brackets yields $103,865.78 of main tax before the $127.20 qualified-dividend tax and $218.42 NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the small retirement contributions and health premiums as eliminating tax on approximately $500,000 of income. Those inputs do not create deductions remotely large enough to erase $472,578.56 of taxable income, and no listed credits offset the resulting liability." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used $472,622 rather than the traced $472,578.56 of taxable income and reported only an understated regular-tax result. It failed to include the $218.42 NIIT and did not complete the required total of main tax plus $127.20 of qualified-dividend tax and NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used roughly $472,622 instead of $472,578.56 and its total omits the full traced tax additions. The requested amount includes $103,865.78 of main tax, $127.20 on qualified dividends, and $218.42 of NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model overstated NIIT as about $332 by using investment income before the $3,000 capital-loss offset. Net investment income is $5,748, so NIIT is $218.42, and taxable income is $472,578.56 rather than the model's estimate." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model drastically underapplied the 2026 married-filing-jointly brackets to taxable income near $472,579. Main income tax alone is $103,865.78, so $57,891 cannot result from the stated income and deduction calculation." -us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used a rough effective-rate guess instead of applying the progressive 2026 joint brackets. Those brackets produce $103,865.78 of main tax, and the two applicable additions raise the total only to $104,211.41." -us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a post-TCJA-expiration regime, using a $16,600 deduction and personal exemptions instead of the $32,200 joint standard deduction and enacted 2026 brackets. It also improperly removed the $16,208 employer premium from gross wages and used $8,748 instead of $5,748 for NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $16,208 employer-sponsored premium from wages and deducted the nondeductible $43 IRA contribution. It also calculated NIIT on $8,748 rather than the $5,748 remaining after the capital-loss offset." -us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model invented a $4,517.80 overtime deduction from annual wages based on the hourly-rate and schedule fields. The prompt states that the $500,000 annual wage total already includes overtime, and the traced computation contains no separate overtime deduction." -us,scenario_052,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The reasoning alternates between $16,100 and $30,800 standard deductions and incorrectly says the $43 IRA contribution reduces taxable income despite the high-income workplace-plan phaseout. Its approximate $104,050 also omits the precise $218.42 NIIT required after computing the main and qualified-dividend taxes." -us,scenario_052,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the entire $29,922 capital loss from AGI rather than applying the annual $3,000 net-capital-loss limit, and it failed to reduce wages by the $926 pre-tax 401(k) contribution. It then submitted $165,473.58 even though its own regular-tax calculation was $85,184.10 and its stated AMT addition was zero." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model's own calculation reached about $107,277 and then submitted $117,285 without a supporting computation. It also used estimated brackets and a $32,600 standard deduction instead of the traced 2026 parameters that yield $103,865.78 of main tax." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model first deducted the entire $29,922 capital loss rather than the $3,000 annual limit, then used a $28,700 standard deduction and finally submitted $103,241 despite its stated recalculations producing different amounts. The traced deductions produce $472,578.56 of taxable income before applying the 2026 brackets." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used approximate bracket thresholds and a $32,300 standard deduction, then added $2,250 of Additional Medicare Tax even though that amount belongs in payroll tax rather than federal income tax before refundable credits. Its final $110,997 was also unsupported by its own intermediate total of about $107,040." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model arithmetically overstated AGI as about $505,748 even though $500,000 minus the $926.16 pre-tax 401(k), plus $7,900 interest, $848 dividends, and the $3,000 capital-loss deduction is about $504,822 before the remaining traced adjustment. That inflated taxable income and main tax above the traced $472,578.56 and $103,865.78." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied the 2026 married-joint brackets incorrectly: taxable income near $472,000 does not generate $141,500 of tax under the stated 10%, 12%, 22%, 24%, and 32% bands. The correct bracket computation is $103,865.78 before the qualified-dividend tax and NIIT additions." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model included $2,250 of Additional Medicare Tax in this income-tax output even though the benchmark assigns it to employee payroll tax. It also used an estimated $30,000 standard deduction instead of $32,200, so both its taxable-income base and tax classification were wrong." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,period_annualization,False,"The prompt explicitly states that all amounts are annual totals, but the model annualized the $926 401(k) and $43 IRA inputs by multiplying them by 52. It then abandoned its own roughly $94,754 calculation and submitted an unsupported $121,500." +us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the $16,208 employer-sponsored insurance premium from the explicitly stated gross annual wages even though the trace only reduces employment income by the $926.16 pre-tax 401(k) contribution. It also used $8,748 as NIIT income instead of netting the $3,000 capital-loss deduction, for the correct $5,748 NIIT base." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a supposed TCJA sunset and estimated $45,620 of unlisted mortgage-interest and SALT deductions despite the instruction that unlisted amounts are zero. The applicable 2026 brackets and $32,200 standard deduction produce $103,865.78 of main tax, not its post-sunset estimate." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's AGI near $476,323 reflects using far more than the permitted $3,000 deduction for the $29,922 net capital loss. The trace limits that loss and reaches $504,778.56 of AGI and $472,578.56 of taxable income." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The submitted $123,753 is inconsistent with applying the traced 2026 married-joint brackets to $472,578.56 of taxable income. That bracket calculation yields $103,865.78, followed by only $127.20 of qualified-dividend tax and $218.42 of NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly assumed post-TCJA-expiration rules, using a $15,300 standard deduction and personal exemptions instead of the applicable $32,200 joint standard deduction. It also wrongly reduced wages by the $16,208 employer premium, compounding the erroneous tax base." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model named the correct broad components but did not apply the traced 2026 brackets to $472,578.56 of taxable income. Those brackets yield $103,865.78 of main tax, and the two stated additions bring the total to $104,211.41 rather than $113,421." +us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model incorrectly treated TCJA provisions as expired and used pre-TCJA rates, a $15,900 standard deduction, and personal exemptions. The applicable 2026 calculation instead uses the $32,200 joint standard deduction and current bracket structure." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The $50,504 answer implies a severe underapplication of the progressive joint brackets to $472,578.56 of taxable income. The main bracket tax alone is $103,865.78 before qualified-dividend tax and NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the small retirement contribution and health-premium inputs as eliminating tax on more than $500,000 of income. The traced deductions leave $472,578.56 taxable, and no nonrefundable credits erase the resulting liability." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model stopped at an approximate regular-tax figure using $472,622 of taxable income and omitted the remaining traced above-the-line adjustment that lowers taxable income to $472,578.56. It also failed to add the $218.42 NIIT to the $103,865.78 main tax and $127.20 qualified-dividend tax." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used approximately $472,622 of taxable income rather than the traced $472,578.56 and did not include the $218.42 NIIT in its stated computation. The complete three-part total is $103,865.78 plus $127.20 plus $218.42." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model overstated NIIT as about $332 by using $8,748 of investment income instead of netting the $3,000 capital-loss deduction. Net investment income is $5,748, so NIIT is $218.42." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The $57,891 answer drastically underapplies the 2026 married-joint brackets to taxable income above $472,000. The main income-tax calculation alone yields $103,865.78." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used a rough effective-tax estimate instead of computing the 2026 married-joint brackets. On $472,578.56 of taxable income, the main tax is $103,865.78, not roughly $140,000." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied projected post-TCJA rules, including pre-TCJA rates, a $16,600 standard deduction, and personal exemptions. It also wrongly subtracted the $16,208 employer premium from wages and failed to net the capital-loss deduction when computing NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted the $16,208 employer insurance premium and the nondeductible $43 IRA contribution from income. It also calculated NIIT on $8,748 rather than the $5,748 remaining after the $3,000 capital-loss deduction." +us,scenario_052,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used $504,822 of AGI and $472,622 of taxable income, omitting the remaining traced above-the-line adjustment that brings those amounts to $504,778.56 and $472,578.56. Applying the exact brackets to the correct base yields $103,865.78 of main tax rather than its approximate $104,035." +us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so the required answer could not be parsed." +us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model invented an overtime deduction from the listed hourly rate and hours even though gross wages already include all overtime and the trace contains no such deduction. This improperly reduced taxable income from $472,578.56 to $468,104.20; it also omitted NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model gave mutually inconsistent standard deductions of $16,100 and $30,800 and treated the nondeductible $43 IRA contribution as reducing taxable wages. The traced calculation uses $3,043.28 of total above-the-line deductions and a $32,200 standard deduction." +us,scenario_052,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the full $29,922 net capital loss instead of limiting the current-year deduction to $3,000, and it omitted the pre-tax 401(k) adjustment. Its submitted $165,473.58 also bears no relationship to its stated regular tax of $85,184.10 and zero AMT." +us,scenario_052,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model used an erroneous shortcut of $76,256 plus 35% of $218,948 rather than applying each 2026 married-joint bracket, and it mishandled the qualified-dividend rate differential. It also omitted the pre-tax 401(k) and other traced above-the-line deductions, overstating AGI and taxable income." us,scenario_052,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_052,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,The model invented a simplified benchmark rule making a 59-year-old Medicare eligible. It missed the controlling Medicare age threshold of 65 and had no listed disability or other qualifying Medicare pathway to support eligibility. -us,scenario_052,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $16,208 employer-sponsored insurance premium from Medicare and Additional Medicare Tax wages even though the input does not establish an employee pre-tax cafeteria-plan deduction. It then submitted $12,820.40, which does not equal either its erroneous $20,558.11 calculation or the correct $20,939 sum." -us,scenario_052,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $168,600 Social Security wage base instead of the 2026 $184,500 base and used the $200,000 employer-withholding threshold instead of the $250,000 joint-return threshold for Additional Medicare Tax. Its submitted $38,335.50 also does not equal its stated $20,403.20 component sum." -us,scenario_052,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a $176,100 Social Security wage base instead of the 2026 $184,500 base. This understated Social Security tax by $520.80 while its Medicare and Additional Medicare Tax calculations were correct." -us,scenario_052,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied the $200,000 single-employer withholding threshold to Additional Medicare Tax instead of the $250,000 married-filing-jointly liability threshold. That produced $2,700 rather than $2,250 of Additional Medicare Tax, overstating the total by $450." -us,scenario_052,payroll_tax,claude-opus-5,llm_error,other,False,"The model stated all three correct components—$11,439, $7,250, and $2,250—and correctly summed them to $20,939, but then submitted $21,118. The error is an unsupported final-value substitution after completing the calculation correctly." -us,scenario_052,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model repeatedly substituted the 2025 Social Security wage base of $176,100 and other projections for the 2026 wage base of $184,500. Using $176,100 understated employee Social Security tax by $520.80." -us,scenario_052,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model alternated between incorrect wage bases and used the $200,000 withholding threshold instead of the $250,000 joint-return threshold, but even its erroneous stated components summed to $20,868.20. The submitted $16,856.10 is not the result of its own component calculation and omits $4,083.90 of the correct tax." -us,scenario_052,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages by the $16,208 employer-sponsored insurance premium without an input establishing a pre-tax employee payroll deduction. It also used a $181,000 Social Security wage base instead of $184,500, understating both the taxable base and total payroll tax." -us,scenario_052,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a projected $171,300 Social Security wage base instead of the 2026 $184,500 base. This produced $10,620.60 rather than $11,439 of Social Security tax, while its two Medicare components were correct." -us,scenario_052,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model calculated Additional Medicare Tax as $18,958.50 instead of applying 0.9% to the $250,000 of wages above the joint threshold, which yields $2,250. It also understated capped Social Security tax as $9,932.50 rather than $11,439." -us,scenario_052,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The submitted total is $660 below the correct $20,939 even though the model named all three required federal components. With Medicare tax fixed at $7,250 and Additional Medicare Tax at $2,250, its number leaves only $10,779 for Social Security rather than the $11,439 generated by the 2026 wage base." -us,scenario_052,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly subtracted the $16,208 employer-sponsored insurance premium from Medicare wages and the Additional Medicare Tax base. It compounded that error by using a $172,200 Social Security wage base instead of the 2026 $184,500 base." -us,scenario_052,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model named the correct tax components and joint Additional Medicare Tax threshold but submitted a total $586 below their correct sum. Its answer therefore applies an understated Social Security wage-base amount or equivalent capped Social Security tax instead of the required $11,439." +us,scenario_052,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a simplified benchmark rule making a 59-year-old Medicare eligible. Medicare's standard age threshold is 65, and the prompt supplies no disability, end-stage renal disease, or other alternative eligibility basis, so the head is not eligible." +us,scenario_052,payroll_tax,claude-fable-5,llm_error,other,False,"The model explicitly derived the correct $20,939 total and then submitted $12,820.40, a number unsupported by either of its displayed FICA calculations. It also incorrectly treated the listed employer-sponsored insurance premium as a pre-tax cafeteria-plan deduction from Medicare wages despite no such treatment being specified." +us,scenario_052,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model used an obsolete $168,600 Social Security wage base and the $200,000 employer-withholding threshold instead of the $250,000 joint-return Additional Medicare Tax threshold. Its submitted $38,335.50 also contradicts its own displayed $20,403.20 sum and has no supporting computation." +us,scenario_052,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used $176,100 instead of the 2026 Social Security wage base of $184,500. That understated Social Security tax by $520.80 while its Medicare and Additional Medicare Tax calculations were correct." +us,scenario_052,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied the $200,000 employer-withholding threshold for Additional Medicare Tax rather than the $250,000 married-filing-jointly liability threshold. This taxed an extra $50,000 at 0.9%, overstating payroll tax by $450." +us,scenario_052,payroll_tax,claude-opus-5,llm_error,other,False,"The model correctly calculated $11,439 of Social Security tax, $7,250 of Medicare tax, and $2,250 of Additional Medicare Tax, which sum to $20,939. It then replaced that exact total with an unsupported rounded submission of $21,118." +us,scenario_052,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model carried the 2025 Social Security wage base of $176,100 into 2026 instead of using the 2026 base of $184,500. Its Medicare and joint-threshold Additional Medicare calculations were correct, so the wrong wage cap accounts for the $520.80 understatement." +us,scenario_052,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model's submitted $16,856.10 does not follow from any calculation in its explanation, which instead displayed totals of $20,868.20 and component inputs supporting $20,939 after using the proper wage base and joint threshold. It also alternated between the obsolete $176,100 Social Security base and the $200,000 employer-withholding threshold." +us,scenario_052,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the $16,208 employer-sponsored insurance premium from FICA wages and used an incorrect $181,000 Social Security wage base. The trace taxes all $500,000 for Medicare, applies Additional Medicare Tax to $250,000, and uses the $184,500 Social Security cap." +us,scenario_052,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a projected $171,300 Social Security wage base instead of the 2026 value of $184,500. Its Medicare and Additional Medicare Tax components were otherwise correct, leaving Social Security tax understated by $818.40." +us,scenario_052,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model calculated $18,958.50 of Additional Medicare Tax instead of 0.9% of the $250,000 above the joint threshold, which is $2,250. It also understated capped Social Security tax, which is $11,439 at the $184,500 wage base." +us,scenario_052,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The submitted total implies that the model understated one or more capped Social Security inputs while naming all three required components. The correct components are $11,439 of Social Security tax, $7,250 of Medicare tax, and $2,250 of Additional Medicare Tax, totaling $20,939." +us,scenario_052,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages by the $16,208 employer-sponsored insurance premium and also used a $172,200 Social Security cap instead of $184,500. Medicare applies to the full $500,000, and Additional Medicare Tax applies to the full $250,000 excess over the joint threshold." +us,scenario_052,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"Although the model named the correct tax components and joint threshold, its $20,353 total omits $586 from their correct amounts. Applying the $184,500 Social Security cap and full $500,000 Medicare wage base produces components of $11,439, $7,250, and $2,250." us,scenario_052,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $926 traditional 401(k) contribution from Social Security and Medicare wages; traditional 401(k) deferrals remain subject to FICA. It also used a $183,000 Social Security wage base instead of $184,500." -us,scenario_052,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model incorrectly stated that $500,000 of wages falls below the $250,000 married-filing-jointly Additional Medicare Tax threshold and omitted the resulting $2,250 tax. Its $38,897 answer also reflects failure to cap Social Security tax at $11,439 rather than summing the specified employee-side components." -us,scenario_052,payroll_tax,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model invented a requirement for an explicit payroll-tax-applicability input and treated $500,000 of wages as exempt. Wage income is subject to employee Social Security and Medicare taxes by default, including Additional Medicare Tax above the joint threshold." -us,scenario_052,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"The model used an obsolete Social Security wage base near $168,000 and then failed to add its own three described components correctly. Even with that obsolete cap, Social Security plus standard and Additional Medicare taxes exceed $20,000, not $13,500." -us,scenario_052,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model incorrectly treated the $16,208 employer-sponsored insurance premium as a pre-tax employee deduction from Medicare wages and the Additional Medicare Tax base. Applying both Medicare rates to the full $500,000 produces $7,250 and $2,250, not $7,015 and $2,104." -us,scenario_052,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced Medicare wages by the $16,208 employer-sponsored insurance premium and used a projected $183,144 Social Security wage base instead of $184,500. Both choices understated the employee payroll-tax total." -us,scenario_052,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax value or explanation. It therefore failed the required structured-output contract without performing a usable computation. -us,scenario_052,payroll_tax,minimax-m3,llm_error,other,False,"The model stated the correct components of $11,439, $7,250, and $2,250, which sum to $20,939, but then claimed a $30,939 subtotal and submitted $11,573. The reference to removing a nonexistent spouse portion is an invalid arithmetic adjustment because the spouse contributed zero wages and zero payroll tax from the outset." -us,scenario_052,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model used the wrong $176,100 Social Security wage base and correctly summed its resulting components to $20,418.20, but submitted $32,846 instead. The submitted value has no support in its stated calculation and exceeds the correct component sum by $11,907." -us,scenario_052,spouse_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a simplified Medicare eligibility rule that treats a 55-year-old spouse as eligible. It missed that Medicare eligibility requires age 65 absent a listed disability, ESRD, ALS, or other qualifying pathway, and the prompt sets unlisted statuses to false." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's submitted $6,224 contradicts its own final calculation of $5,335.60. It also improperly subtracted the separately listed $3,389 employer-sponsored premium from gross wages and introduced an auto-loan-interest deduction even though the $16,100 standard deduction governs taxable income." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction instead of $16,100 and incorrectly removed $3,389 of employer-sponsored premiums from the listed gross wages. Its stated bracket arithmetic also produces $12,761.60, not the submitted $5,744." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model substituted an estimated $16,600 standard deduction for the applicable $16,100 amount, then abandoned its own approximately $5,796 bracket calculation and submitted $6,196. The correct $50,868.68 taxable income produces $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly derived approximately $5,903 from the $16,100 deduction and 2026 brackets, then replaced that result with $6,027 without any supporting computation. The submitted value therefore contradicts its own completed calculation." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly computed approximately $5,903 using the applicable deduction and bracket segments, then submitted $6,379. No computation in its explanation supports the submitted amount." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model repeatedly substituted projected or 2025 parameters, ultimately using a $15,700 standard deduction and estimated bracket thresholds instead of the applicable $16,100 deduction and 2026 brackets. That left taxable income and tax overstated." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used projected bracket thresholds derived by inflating earlier-year figures instead of the applicable 2026 thresholds. With taxable income of $50,868.68, the actual bracket computation yields $5,903.11 rather than $6,154." -us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the separately listed $3,389 employer-sponsored premium from gross wages and used a $15,400 standard deduction instead of $16,100. PolicyEngine starts from $66,968.68 of employment income and arrives at $50,868.68 of taxable income." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a TCJA-sunset regime with an $8,300 standard deduction, a personal exemption, and 10%, 15%, and 25% brackets. The applicable 2026 law uses the $16,100 standard deduction and current 10%, 12%, and 22% bracket structure at this income." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $3,939 is inconsistent with applying the 2026 single-filer brackets to wages after a standard deduction. The correct $16,100 deduction leaves $50,868.68 taxable and produces $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration and applied a pre-TCJA tax regime. It also subtracted the separately listed $3,389 employer premium from gross wages; the applicable computation uses $66,968.68 of AGI, a $16,100 standard deduction, and the 2026 current-law brackets." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly used a sunset-law personal exemption, an $8,300 standard deduction, and 15% and 25% marginal brackets. The applicable $16,100 standard deduction and 10%, 12%, and 22% bracket schedule produce $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the separately listed $3,389 employer-sponsored premium from the stated gross wages. The engine treats $66,968.68 as employment income, so the $16,100 standard deduction leaves $50,868.68 taxable rather than $47,480." -us,scenario_053,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used the obsolete $14,600 standard deduction and older bracket thresholds of $11,600 and $47,150. The applicable $16,100 deduction and 2026 thresholds reduce the tax to $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model invoked a basic CTC or other nonrefundable credits despite the household having no dependents and no qualifying credit facts. No nonrefundable credit reduces the bracket tax, so the full $5,903.11 remains before refundable credits." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model's approximate $5,118 does not result from the applicable $16,100 standard deduction and 2026 single brackets. Those parameters leave $50,868.68 taxable and yield $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model approximated the standard deduction as $16,000 and then misapplied the 2026 brackets: even its stated $50,969 taxable income does not support $5,793. Using the exact $16,100 deduction and bracket thresholds produces $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the correct rounded taxable income but miscalculated the bracket tax as $5,985. Applying the 10%, 12%, and 22% segments to $50,868.68 produces $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an approximate $15,000 standard deduction instead of $16,100 and supplied a tax number that does not follow from the applicable brackets. The correct taxable income is $50,868.68 and the resulting tax is $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied a TCJA-sunset system with an $8,300 standard deduction, a $5,300 personal exemption, and 15% and 25% brackets. The applicable 2026 current-law parameters use a $16,100 standard deduction and produce $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly used pre-TCJA law, including an $8,372 standard deduction and the 10%, 15%, and 25% rate structure. It also omitted the applicable deduction regime that leaves $50,868.68 taxable under the 2026 current-law brackets." -us,scenario_053,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration and used an $8,300 standard deduction, a personal exemption, and pre-TCJA rates. It additionally subtracted $3,389 of employer premiums from gross wages; the applicable calculation instead uses $66,968.68 of AGI and a $16,100 standard deduction." -us,scenario_053,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model used a $13,850 standard deduction instead of $16,100 and calculated $6,397.98 from that error, but then submitted the unrelated value $4,272. The answer therefore fails both the parameter calculation and consistency between explanation and output." -us,scenario_053,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own bracket calculation produced approximately $6,127, but it submitted $8,252 with no supporting step. It also used projected prior-year thresholds and a $16,000 deduction instead of the exact $16,100 deduction and applicable 2026 brackets." -us,scenario_053,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the $3,389 employer-sponsored insurance premium from FICA wages. PolicyEngine applies Social Security and Medicare taxes to the full wage base, yielding $4,152.06 plus $971.05 rather than taxes on $63,580." -us,scenario_053,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed employer-sponsored insurance premium as a pre-tax payroll deduction even though the prompt does not establish that treatment. Applying 7.65% to the reduced $63,580 base omitted payroll tax due on $3,389 of wages." -us,scenario_053,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages by the $3,389 employer-sponsored insurance premium. The full payroll-tax wage base produces $4,152.06 of Social Security tax and $971.05 of Medicare tax." -us,scenario_053,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model assumed without support that the employer-sponsored health premium was excluded from payroll-tax wages. That reduced both its Social Security and Medicare components, whereas the traced taxes are $4,152.06 and $971.05 on the full applicable wage base." -us,scenario_053,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model misclassified the $3,389 insurance premium as a pre-tax FICA deduction and taxed only $63,580. PolicyEngine does not subtract that input from the payroll-tax base, so the model understated both employee FICA components." -us,scenario_053,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model selected the correct 6.2% and 1.45% employee FICA rates and used gross wages, but its final arithmetic is wrong. The traced component calculation yields $4,152.06 of Social Security tax plus $971.05 of Medicare tax, totaling $5,123.10 rather than $5,124.74." -us,scenario_053,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's stated formula does not produce its submitted answer: 7.65% of $66,969 is approximately $5,123.13 before engine-level rounding, not $5,329. The traced components total $5,123.10, so the error is arithmetic rather than an Additional Medicare Tax or Idaho payroll-tax rule." -us,scenario_053,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"The model incorrectly excluded the $3,389 employer-sponsored insurance premium from FICA wages. The absence of Additional Medicare Tax and Idaho employee payroll taxes is correctly identified, but Social Security and Medicare must be computed on the full applicable wage base, producing $5,123.10." +us,scenario_052,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model incorrectly stated that $500,000 of wages were below the $250,000 married-filing-jointly Additional Medicare Tax threshold and therefore omitted the $2,250 tax. Its $38,897 submission also exceeds the correctly capped employee Social Security and Medicare components and is unsupported by its explanation." +us,scenario_052,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model treated the absence of a separate payroll-tax-applicability flag as exempting explicitly listed wages from payroll tax. The $500,000 of wages directly triggers employee Social Security, Medicare, and Additional Medicare taxes totaling $20,939." +us,scenario_052,payroll_tax,grok-4.3,llm_error,other,False,"The model's approximate $13,500 total does not equal the components it claimed to include. Using the $184,500 Social Security cap, full $500,000 Medicare base, and $250,000 joint-threshold excess yields $11,439, $7,250, and $2,250, respectively." +us,scenario_052,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model improperly subtracted the $16,208 employer-sponsored insurance premium from Medicare wages and from wages subject to Additional Medicare Tax. Both taxes use the full $500,000 wage amount here, producing $7,250 and $2,250 rather than $7,015 and $2,104." +us,scenario_052,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages by the $16,208 insurance premium and used a projected $183,144 Social Security wage base instead of $184,500. The full wages generate $7,250 of Medicare tax and $2,250 of Additional Medicare Tax, while capped Social Security tax is $11,439." +us,scenario_052,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required output contract." +us,scenario_052,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly listed components of $11,439, $7,250, and $2,250 but then added them incorrectly as $30,939 and submitted an unrelated $11,573. With no spouse wages to subtract, those components sum directly to $20,939." +us,scenario_052,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model repeatedly calculated $20,418.20 from its stated components but submitted the unrelated value $32,846. It also used the obsolete $176,100 Social Security base; replacing it with $184,500 raises the correct total to $20,939." +us,scenario_052,payroll_tax,qwen3.8-max,llm_error,other,False,"The model double-counted the $2,250 Additional Medicare Tax by first including it in the stated $20,418.20 subtotal and then adding it again. It also used the obsolete $176,100 Social Security wage base instead of $184,500, so the correctly computed single-count total is $20,939." +us,scenario_052,spouse_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a simplified rule that makes a 55-year-old Medicare eligible. The spouse is below the age-65 threshold and has no listed disability or other qualifying Medicare pathway, yielding spouse_medicare_eligible = 0." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model improperly subtracted $3,389 of employer premiums from wages even though the supplied gross wage amount is the tax input and also introduced an auto-loan-interest deduction that does not affect this reference computation. Its submitted $6,224 additionally contradicts its own final calculation of about $5,336." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model deducted employer premiums from the supplied wages and used a $14,600 standard deduction instead of $16,100. It also wrote an internally invalid bracket calculation and then submitted $5,744, a number unsupported by its stated $12,761.60 result." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $16,600 standard deduction instead of $16,100. More decisively, its own bracket arithmetic produced about $5,796, but it replaced that result with an unexplained $6,196 submission." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly reached approximately $5,903 from $50,869 of taxable income and the applicable brackets. It then discarded that calculation and submitted $6,027 without any corresponding adjustment, tax, or credit." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly calculated approximately $5,903 using the $16,100 deduction and the applicable 10%, 12%, and 22% bracket segments. Its $6,379 output is an unsupported substitution that contradicts every component of its own derivation." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model repeatedly substituted projected or 2025 parameters for the applicable 2026 values, including standard deductions from $15,000 to $15,700 and bracket cutoffs of $11,925 and $48,475. The correct $16,100 deduction and 2026 brackets applied to $50,868.68 yield $5,903.11, not $6,193." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Although it used the correct approximate taxable income, the model inflated projected 2025 bracket thresholds by an assumed 2% instead of applying the specified 2026 brackets. That parameter substitution produced $6,154 rather than the tax of $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted $3,389 of employer-sponsored premiums from the supplied annual wages and used a $15,400 standard deduction instead of $16,100. The trace treats $66,968.68 as AGI, so taxable income is $50,868.68 rather than $48,180." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset regime with an $8,300 standard deduction, a $5,300 personal exemption, and 15% and 25% brackets. It also deducted employer premiums from wages; the applicable 2026 law uses the $16,100 standard deduction and 10%, 12%, and 22% bracket segments relevant here." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model cited a $15,300 estimated standard deduction instead of the applicable $16,100 amount and supplied no bracket arithmetic capable of producing $3,939. Applying the 2026 brackets to $50,868.68 of taxable income yields $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model wrongly applied a pre-TCJA 2026 regime and deducted employer premiums from the supplied wage amount. The applicable computation uses $66,968.68 as AGI, a $16,100 standard deduction, and the continuing 10%, 12%, and 22% bracket structure at this income." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration and applied an $8,300 standard deduction, a $5,300 personal exemption, and 15% and 25% marginal rates. It also reduced wages by employer premiums; the correct taxable income is $50,868.68 under the applicable 2026 parameters." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the supplied $66,969 wage amount by $3,389 of employer premiums. With wages retained as AGI and the $16,100 standard deduction, taxable income is $50,868.68 rather than $47,480, producing $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an obsolete $14,600 standard deduction and obsolete bracket thresholds of $11,600 and $47,150. The applicable $16,100 deduction and 2026 thresholds reduce taxable income to $50,868.68 and tax to $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model invoked a basic CTC or other nonrefundable credits despite the household having no child, dependent, or other stated credit qualification. No nonrefundable credit applies, and ordinary bracket tax after the $16,100 standard deduction remains $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model relied on unspecified approximations for both the standard deduction and 2026 brackets. The exact inputs are a $16,100 deduction and $50,868.68 of taxable income, whose bracket tax is $5,903.11 rather than $5,118." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an estimated $16,000 standard deduction instead of $16,100 and then misapplied the brackets: even its stated $50,969 taxable income does not support $5,793. The exact taxable income of $50,868.68 produces $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the correct approximate taxable income but calculated the 10%, 12%, and 22% bracket segments incorrectly. Those segments sum to $5,903.11, not $5,985." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an approximate $15,000 standard deduction rather than $16,100 and did not perform an exact bracket calculation. The resulting taxable income is $50,868.68, and the applicable brackets yield $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly imposed a TCJA-sunset system with an $8,300 standard deduction, a $5,300 personal exemption, and 15% and 25% rates. The applicable 2026 computation instead uses a $16,100 standard deduction and produces $50,868.68 of taxable income taxed at the relevant 10%, 12%, and 22% segments." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied pre-TCJA law, used only an $8,372 standard deduction, and taxed $58,597 under 10%, 15%, and 25% brackets. The applicable deduction is $16,100 and taxable income is $50,868.68 under the operative 2026 brackets." +us,scenario_053,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration, deducted employer premiums from wages, and used an $8,300 standard deduction plus a $5,300 personal exemption with 15% and 25% rates. The applicable calculation starts from the full wage amount and uses the $16,100 standard deduction and current 2026 bracket structure." +us,scenario_053,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model mislabeled a $13,850 standard deduction as an adjustment to AGI and calculated tax of $6,397.98 using that obsolete amount. It then submitted $4,272, which has no connection to its own arithmetic; the correct $16,100 deduction yields $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's approximate parameters produced a stated tax of $6,127.18, but it submitted $8,252 without identifying any additional tax or adjustment. The trace contains no such additional tax, and exact 2026 parameters yield $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model reduced regular tax by a child or dependent credit even though this single-person household has no dependent and qualifies for no such nonrefundable credit. It also used unexplained taxable income of $49,779 instead of $50,868.68; tax therefore remains $5,903.11 before refundable credits." +us,scenario_053,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a refundable child or other-dependent credit despite the household containing no qualifying child or dependent, and it also treated this high-income childless filer as receiving refundable EITC. Neither eligibility pathway applies, so the claimed $1,966 credit has no qualifying component." +us,scenario_053,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $3,389 employer-sponsored insurance premium from FICA wages. Applying Social Security and Medicare taxes to the full $66,969 produces $5,123.10, not $4,863.87." +us,scenario_053,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed employer-sponsored insurance premium as a pre-tax payroll deduction and reduced FICA wages to $63,580. PolicyEngine applies employee Social Security and Medicare taxes to the full $66,969 wage amount." +us,scenario_053,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly assumed the employer-sponsored insurance premium reduces payroll-taxable wages. The applicable wage base is $66,969, yielding $4,152.06 of Social Security tax plus $971.05 of Medicare tax." +us,scenario_053,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model deducted $3,389 of employer-sponsored insurance premiums from the FICA base without an input establishing that the amount reduces taxable wages. Social Security and Medicare taxes instead apply to all $66,969 of wages, producing $5,123.10." +us,scenario_053,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model reduced FICA wages from $66,969 to $63,580 by treating the insurance premium as pre-tax. Using the full wage base gives $4,152.06 of Social Security tax and $971.05 of Medicare tax." +us,scenario_053,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model selected the correct full wage base and FICA rates but performed the computation incorrectly: 6.2% plus 1.45% on $66,969 does not equal $5,124.74. The engine's separately calculated components total $5,123.10." +us,scenario_053,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model's $5,329 answer is inconsistent with its own stated 6.2% Social Security and 1.45% Medicare rates on $66,969. Those component calculations yield $4,152.06 and $971.05, totaling $5,123.10." +us,scenario_053,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"The model incorrectly classified the $3,389 employer-sponsored insurance premium as a pre-tax reduction to FICA wages. Payroll taxes apply to the full $66,969, while the correctly identified Additional Medicare Tax and Idaho employee payroll-tax amounts remain zero." us,scenario_053,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used the obsolete 5.695% rate and then abandoned its own taxable-income calculations for an unsupported $53,000 tax base. Idaho taxable income is $50,868.68, and the 2026 rate and zero-tax bracket calculation yields $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model applied an obsolete multi-bracket Idaho schedule, an incorrect $7,200 state deduction, and an invented $59 personal-exemption credit. Idaho instead deducts $16,100 and applies the 2026 flat-rate structure to $50,868.68." -us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model guessed among 5.3%, 5.695%, and 4.7% and selected $2,548 without a valid computation. The correct $16,100 deduction produces $50,868.68 of taxable income, to which Idaho’s 2026 rate and zero-tax bracket yield $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used the obsolete 5.695% rate and then reported $2,867 even though its displayed formula produced about $2,631. Applying the 2026 Idaho tax structure to $50,868.68 yields $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model improperly treated the Idaho grocery credit as a nonrefundable offset in this output and did not compute the indexed zero-tax bracket correctly. The pre-refundable-credit liability from $50,868.68 of taxable income is $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model deducted the $3,389 employer-sponsored premium from wages even though the stated gross wages feed AGI of $66,968.68, and it used an obsolete 5.695% rate. The $16,100 deduction leaves $50,868.68 taxable, producing $2,435.28 under the 2026 Idaho structure." -us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model’s stated calculation gives about $2,960, but it submitted $3,706 without any computation supporting that amount. The actual calculation uses $50,868.68 of taxable income and produces $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an unexplained taxable-income base of $48,180 instead of $50,868.68 and treated the 5.3% rate as applying from the first dollar. Idaho’s zero-tax bracket must also be applied, yielding $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied the obsolete 5.695% rate directly to an incorrect $49,980 base. Idaho’s $16,100 deduction leaves $50,868.68 taxable, and the 2026 rate plus zero-tax bracket produces $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $2,652 estimate does not implement Idaho’s 2026 rate and zero-tax bracket. Applying that structure to $50,868.68 of taxable income yields $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used an obsolete 5.692% rate and an unsupported $49,852 taxable-income estimate. The correct base is $50,868.68, and Idaho’s 2026 rate and zero-tax bracket yield $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced AGI by the $3,389 employer-sponsored premium and then applied an obsolete 5.8% rate directly to its tax base. AGI is $66,968.68, the $16,100 deduction leaves $50,868.68 taxable, and the correct liability is $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $3,389 employer-sponsored premium from the stated wages, lowering taxable income to $47,480, and used the obsolete 5.695% rate. The correct taxable income is $50,868.68 and the 2026 Idaho calculation yields $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented a deduction equal to $1,250 per $1,000 of income and therefore erased the entire tax base. Idaho allows a $16,100 deduction here, leaving $50,868.68 taxable and $2,435.28 of tax." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model’s rough $2,050 estimate understates the liability and supplies no Idaho bracket computation. The $50,868.68 taxable base processed through Idaho’s 2026 rate and zero-tax bracket yields $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $1,195 estimate does not correspond to Idaho’s applicable 2026 tax structure. Taxable income is $50,868.68, and the rate and zero-tax bracket produce $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used a $15,750 deduction instead of $16,100 and a $4,489 zero-tax bracket instead of the applicable indexed threshold. Those errors produced $2,476.69 rather than the $2,435.28 liability on $50,868.68 of taxable income." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied 5.3% to every dollar of taxable income and omitted Idaho’s zero-tax bracket. Applying the threshold before the rate to $50,868.68 yields $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model followed the right general flat-rate-minus-zero-bracket method but used an incorrect approximate indexed threshold, producing $2,433.92. The exact 2026 parameters applied to $50,868.68 yield $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model applied 5.3% directly to all $50,869 of taxable income and omitted Idaho’s zero-tax bracket. Applying the threshold first yields $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an obsolete approximate 5.8% rate and an overstated taxable-income base of about $52,000. The correct base is $50,868.68, and Idaho’s 2026 rate and zero-tax bracket produce $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $8,300 deduction instead of Idaho’s $16,100 standard deduction and applied the obsolete 5.695% rate from the first dollar. The correct taxable income is $50,868.68 and the liability is $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $58,597 taxable-income base, an obsolete 5.8% rate, and omitted the zero-tax bracket. Idaho’s $16,100 deduction leaves $50,868.68 taxable and produces $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the output contract." -us,scenario_053,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly reached about $50,869 of taxable income but applied 5.3% to the entire amount, omitting Idaho’s zero-tax bracket. Applying the threshold before the rate yields $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model stated that $66,969 taxed at 5.695% equals about $3,814, then submitted $2,878 with no deduction or credit calculation connecting the two figures. The actual $16,100 deduction leaves $50,868.68 taxable and produces $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced wages by the $3,389 employer-sponsored premium, used an obsolete approximate 5.8% rate, and never specified the deduction that transformed its tentative $3,688 into $2,702. The correct taxable income is $50,868.68 and the 2026 Idaho calculation yields $2,435.28." -us,scenario_053,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,The model misclassified Idaho's grocery credit as nonrefundable and incorrectly imposed an income-based exclusion. The full-year resident receives the refundable $155 credit without a phase-out at the stated wages. -us,scenario_053,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,The model overlooked Idaho's refundable grocery credit and incorrectly treated the absence of a state EITC as establishing zero refundable credits. This resident qualifies for the full $155 grocery credit. -us,scenario_053,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,The model incorrectly netted the grocery credit against state tax liability and concluded that none counted as refundable. PolicyEngine classifies the qualifying full-year grocery credit as a $155 refundable credit regardless of whether tax liability exceeds it. -us,scenario_053,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,The model wrongly treated the grocery credit as nonrefundable in effect and also used an obsolete $120 amount. The 2026 refundable base for this qualifying full-year resident is $155. -us,scenario_053,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,The model incorrectly treated Idaho's grocery credit as only an offset against liability and imposed an income limitation. The household qualifies for the full refundable 2026 amount of $155. -us,scenario_053,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model misclassified Idaho's grocery credit as nonrefundable. It is included in state refundable credits, and this full-year resident receives the $155 base amount." -us,scenario_053,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,The model incorrectly excluded the grocery credit on refundability and income grounds. The credit contributes the full refundable $155 for this qualifying Idaho resident. -us,scenario_053,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model imposed an income limit that does not reduce this household's Idaho grocery credit. All 12 months qualify, producing the full refundable $155 amount." -us,scenario_053,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model identified the correct grocery-credit pathway but used a stale $120 per-person amount. The applicable 2026 base amount is $155. -us,scenario_053,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model omitted Idaho's refundable grocery credit. The head qualifies for all 12 months and receives the full $155 base amount. -us,scenario_053,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly identified the refundable grocery credit but applied an outdated $120 amount. Idaho's 2026 base credit for this resident is $155. -us,scenario_053,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model correctly applied the grocery-credit pathway but used an obsolete $100 amount. The 2026 credit for one qualifying full-year resident is $155. -us,scenario_053,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model identified the refundable grocery credit but used the wrong per-resident parameter of $120. The applicable 2026 base is $155. -us,scenario_053,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,The model misclassified Idaho's grocery credit as nonrefundable. The full-year qualifying resident receives $155 in refundable state credits. -us,scenario_053,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model's zero answer omits Idaho's grocery credit. Full-year Idaho residence supplies the qualifying pathway, and the resulting refundable credit is $155." -us,scenario_053,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,The model looked only for child-related refundable credits and omitted Idaho's resident grocery credit. No child is required for the head to receive the full $155 amount. -us,scenario_053,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model selected the correct refundable grocery credit but used the wrong $120 parameter. The 2026 base amount for one qualifying resident is $155. -us,scenario_053,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model incorrectly disqualified the resident from the grocery credit based on income. The stated wages cause no phase-out or reduction, so the refundable credit is $155." -us,scenario_053,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,The model correctly found that the resident qualifies without a phase-out but used an outdated $120 credit amount. The applicable 2026 base is $155. -us,scenario_053,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The model omitted Idaho's refundable grocery credit. This full-year resident qualifies for $155. -us,scenario_053,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model identified the correct refundable grocery credit but applied an obsolete $100 amount. The 2026 amount for this qualifying nonelderly resident is $155. -us,scenario_053,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model's zero answer omits the Idaho grocery credit. The resident qualifies for all 12 months, yielding $155." -us,scenario_053,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, violating the required output contract." -us,scenario_053,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model correctly applied full-year eligibility with no monthly reduction but used the wrong $120 base amount. Idaho's 2026 base credit is $155. -us,scenario_053,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model incorrectly searched for child and property-tax-credit conditions instead of applying Idaho's grocery credit. The credit does not require children or property-tax eligibility, and this resident receives $155." -us,scenario_053,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model treated the absence of Idaho EITC and child tax credits as proof that no refundable credit applies. It omitted the grocery credit, which provides this full-year resident $155." -us,scenario_054,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model applied only a CHIP upper-income threshold and treated income below that ceiling as sufficient. It failed to test the Medicaid pathway first: Child 1 qualifies under the OLDER_CHILD Medicaid category and is therefore excluded from CHIP. -us,scenario_054,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated low income and lack of other coverage as sufficient for CHIP without applying the prerequisite that the child be ineligible for Medicaid. Child 1 is Medicaid-eligible under the OLDER_CHILD category, which makes CHIP eligibility false." -us,scenario_054,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model used low household income and absence of employer insurance as the complete CHIP test. It omitted the Medicaid-eligibility exclusion: Child 1 qualifies for Medicaid as an older child and therefore cannot qualify for CHIP. -us,scenario_054,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model recognized that income falls below the Medicaid/CHIP boundary but then incorrectly assigned CHIP rather than Medicaid. At this income, Child 1 qualifies under the OLDER_CHILD Medicaid category, and existing Medicaid eligibility bars CHIP eligibility." -us,scenario_054,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated combined Medicaid/CHIP coverage eligibility with CHIP eligibility specifically. Child 1's low income produces Medicaid eligibility under the OLDER_CHILD category, and that Medicaid eligibility excludes the child from CHIP." -us,scenario_054,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stated the controlling condition—CHIP applies only if the child is not Medicaid-eligible—but never evaluated it and returned the opposite result. Child 1 is Medicaid-eligible under the OLDER_CHILD category, so CHIP eligibility is false." -us,scenario_054,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated missing citizenship, coverage, and household-qualifier details as affirmative missing prerequisites even though the prompt instructs that unlisted boolean/status facts are false and PolicyEngine applies program rules from the listed household facts. It never applied North Carolina's older-child Medicaid MAGI pathway for a 10-year-old dependent with MAGI at 0.58 FPL, which makes child1 Medicaid eligible." -us,scenario_054,child1_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a WIC pathway for at-risk school-age children ages 6-18 and treated the head's WIC receipt as evidence that Child 1 could qualify. WIC does not cover a 10-year-old child under the child categorical eligibility rule, so income eligibility and another household member's WIC receipt do not make Child 1 eligible." -us,scenario_054,child2_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated income below the CHIP ceiling as sufficient and failed to apply CHIP's exclusion for children already eligible for Medicaid. At age 2 and this income, Child 2 qualifies for Medicaid as a YOUNG_CHILD and is therefore ineligible for CHIP." -us,scenario_054,child2_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated the combined Medicaid/CHIP income framework with CHIP eligibility and incorrectly applied the CHIP limit directly. Child 2 first qualifies for Medicaid under the YOUNG_CHILD pathway, which excludes the child from CHIP regardless of employer-sponsored insurance." -us,scenario_054,child2_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stated the controlling condition—CHIP applies only if the child is not Medicaid-eligible—but never evaluated it. Child 2 is Medicaid-eligible in the YOUNG_CHILD category, so the condition fails and CHIP eligibility is No." -us,scenario_054,child2_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model focused on the CHIP income limit and affordability of employer-sponsored coverage instead of first applying Medicaid eligibility. Child 2 qualifies for Medicaid as a YOUNG_CHILD, which categorically excludes CHIP eligibility; ESI affordability is not the deciding step." -us,scenario_054,child2_early_head_start_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated gross self-employment income of $17,100 and bank assets as enough to fail Early Head Start income eligibility, instead of using PolicyEngine's computed household income measure, which is approximately $0 and below 100% of the federal poverty line. It also introduced an assets barrier that is not part of the referenced Early Head Start eligibility calculation for this output." -us,scenario_054,child2_head_start_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model stopped after checking age 2 and low income, treating those as sufficient for preschool Head Start eligibility. PolicyEngine applies additional Head Start eligibility conditions beyond the income threshold, and Child 2 fails that full eligibility test, so the correct benchmark output is 0." -us,scenario_054,child2_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model misclassified age 2 as within the preschool Head Start eligibility range and did not apply the full PolicyEngine Head Start eligibility rule. Child 2 does not satisfy the benchmark's preschool Head Start eligibility conditions, so the correct output is 0." -us,scenario_054,child2_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model incorrectly treated employer-sponsored insurance as a Medicaid disqualifier for a North Carolina 2-year-old. It skipped the young-child MAGI pathway, under which the child’s 0.58 x FPL household income satisfies the Medicaid income test." -us,scenario_054,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model failed to use age 2 as the categorical basis for the young-child Medicaid pathway and incorrectly required additional eligibility indicators. Applying the young-child MAGI category to the household income level of 0.58 x FPL yields Medicaid eligibility for Child 2. -us,scenario_054,child2_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated WIC receipt as a required child-specific input and failed to evaluate the requested eligibility rules. It missed that a 2-year-old child is categorically eligible for WIC and that the household's income is below North Carolina's 185% FPL income threshold, so explicit WIC receipt by Child 2 is not needed." +us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model abandoned the correctly identified $50,869 taxable-income base and instead applied 5.695% to an invented $53,000 base. Idaho’s 2026 bracket calculation on $50,868.68 yields $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used obsolete progressive brackets topping out at 7.15% and an incorrect $7,200 state standard deduction. The applicable $16,100 deduction produces $50,868.68 of taxable income, to which the 2026 Idaho bracket rules yield $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model guessed among 5.3%, 5.695%, and 4.7% instead of applying the enacted 2026 Idaho bracket parameters. It also used a $15,750 deduction rather than $16,100, so it never computed the $50,868.68 taxable-income base and $2,435.28 liability." +us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used the obsolete 5.695% rate and then reported $2,867 even though its own stated formula produced about $2,631. The 2026 bracket calculation on $50,868.68 produces $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly treated the Idaho grocery credit as a nonrefundable offset in this output and did not execute the 2026 bracket calculation. Applying the brackets to $50,868.68 before refundable credits yields $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the $3,389 employer-sponsored insurance premium from the stated wages, reducing AGI to $63,580. PolicyEngine’s AGI is $66,968.68; after the $16,100 deduction, Idaho taxes $50,868.68 under its 2026 brackets to produce $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The submitted $3,706 contradicts the model’s own stated calculation of about $2,960 and has no supporting computation. The required derivation uses $50,868.68 of taxable income and the 2026 Idaho brackets, producing $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an unexplained taxable-income figure of $48,180 instead of subtracting the $16,100 standard deduction from $66,968.68. The correct $50,868.68 base, processed through Idaho’s 2026 brackets, yields $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model multiplied its taxable-income estimate by the obsolete 5.695% rate across the entire base. Idaho’s 2026 bracket structure, including the zero-tax threshold, applied to $50,868.68 yields $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies a rough flat-rate estimate rather than the 2026 Idaho bracket computation. The $16,100 deduction leaves $50,868.68 taxable, and the applicable brackets yield $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied 5.692% to nearly all taxable income instead of using Idaho’s 2026 bracket structure and zero-tax threshold. The brackets applied to $50,868.68 produce $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced AGI by the $3,389 employer-sponsored insurance premium and then used a 5.8% flat rate. PolicyEngine starts from $66,968.68 of AGI, subtracts $16,100, and applies the 2026 brackets to obtain $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the $3,389 employer-sponsored insurance premium from AGI before taking the standard deduction. AGI remains $66,968.68, leaving $50,868.68 taxable and $2,435.28 under the 2026 Idaho brackets." +us,scenario_053,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented a deduction equal to $1,250 per $1,000 of federal taxable income, producing the impossible $83,711.25 deduction. Idaho deductions are $16,100 here, leaving $50,868.68 taxable and generating $2,435.28 of tax." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model supplied a rough $2,050 estimate without identifying or applying Idaho’s 2026 brackets. Those brackets applied to $50,868.68 of taxable income yield $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $1,195 answer reflects neither the 2026 Idaho bracket calculation nor the tax generated by the $50,868.68 taxable-income base. Applying the required brackets yields $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used a $15,750 standard deduction and a $4,489 zero bracket instead of the applicable $16,100 deduction and 2026 Idaho parameters. The correct taxable income is $50,868.68, and its bracket tax is $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied 5.3% to every dollar of taxable income, omitting Idaho’s zero-tax bracket. Applying the full 2026 bracket structure to $50,868.68 yields $2,435.28, not $2,696.06." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used an incorrect approximate zero-tax threshold in an otherwise close 5.3% calculation. Idaho’s exact 2026 bracket parameters applied to $50,868.68 yield $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model taxed the entire $50,869 base at 5.3%, omitting Idaho’s zero-tax bracket. The complete 2026 bracket calculation yields $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used a rough $52,000 base and an obsolete approximate 5.8% flat rate. The $16,100 deduction leaves $50,868.68 taxable, and Idaho’s 2026 brackets yield $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $8,300 standard deduction, overstating taxable income as $58,669. The applicable $16,100 deduction leaves $50,868.68, whose 2026 Idaho bracket tax is $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an unstated deduction that left $58,597 taxable and then applied an obsolete 5.8% rate. The correct deduction is $16,100, producing $50,868.68 of taxable income and $2,435.28 under the 2026 brackets." +us,scenario_053,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied 5.3% to the entire taxable-income base and omitted Idaho’s zero-tax bracket. The full bracket calculation on $50,868.68 yields $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the submission contract." +us,scenario_053,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model applied 5.3% to all $50,869 of taxable income, omitting Idaho’s zero-tax bracket. Applying the complete 2026 bracket structure yields $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The submitted $2,878 does not follow from the model’s stated $3,814 calculation, and the reasoning also claimed no deduction despite the applicable $16,100 standard deduction. The correct base is $50,868.68 and the 2026 Idaho bracket tax is $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced wages by the $3,389 employer-sponsored insurance premium and used an approximate 5.8% rate before applying an unspecified deduction. PolicyEngine’s AGI is $66,968.68; subtracting $16,100 and applying the 2026 brackets yields $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model gave $1,588 without a concrete deduction, bracket, rate, or credit calculation supporting it. The trace produces $50,868.68 of taxable income, and Idaho’s 2026 brackets yield $2,435.28 before refundable credits." +us,scenario_053,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,The model misclassified Idaho's grocery credit as nonrefundable and incorrectly treated income and the absence of children as disqualifying. The full-year resident receives the entire $155 grocery credit in state refundable credits. +us,scenario_053,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model overlooked Idaho's refundable grocery credit by focusing on the absence of a state EITC and asserting that income precluded all credits. This resident qualifies for the full-year $155 grocery credit without an income reduction. +us,scenario_053,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,The model incorrectly netted the grocery credit against state tax liability and reported only an excess refund as a refundable credit. PolicyEngine classifies the full $155 grocery credit in state refundable credits even when tax liability exceeds it. +us,scenario_053,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated the grocery credit as effectively nonrefundable because the taxpayer has positive liability. The output records the full $155 grocery credit as a refundable state credit, not merely an amount exceeding liability." +us,scenario_053,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,The model incorrectly treated the grocery credit as only an offset against liability and concluded that no refundable amount remained. The full-year grocery credit contributes $155 directly to state refundable credits. +us,scenario_053,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly classified Idaho's grocery credit as nonrefundable. For this full-year qualifying resident, it contributes the full $155 to state refundable credits." +us,scenario_053,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,The model incorrectly limited refundability to an amount left after offsetting liability and treated income and dependents as controlling eligibility. Idaho's grocery-credit pathway supplies a $155 refundable credit for this resident. +us,scenario_053,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model invented an income limit that disqualified the resident from Idaho's grocery credit. No phase-out or reduction applies here, so the full-year credit is $155." +us,scenario_053,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model identified the correct grocery-credit pathway but used a $120 per-person amount instead of the 2026 base of $155. Full-year qualification therefore yields $155. +us,scenario_053,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The zero answer omitted Idaho's grocery credit for a full-year qualifying resident. That pathway alone produces $155 in state refundable credits. +us,scenario_053,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly identified the refundable grocery credit but applied an obsolete $120 amount. The applicable 2026 nonelderly base is $155. +us,scenario_053,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used a $100 grocery-credit amount instead of Idaho's 2026 base amount. One resident qualifying for all 12 months receives $155. +us,scenario_053,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model applied a $120 grocery-credit parameter rather than the 2026 $155 base. Full-year qualification makes the final credit $155. +us,scenario_053,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,The model incorrectly classified Idaho's grocery credit as nonrefundable and demanded additional facts to establish eligibility. Idaho residency and full-year qualification produce a $155 refundable grocery credit. +us,scenario_053,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model omitted Idaho's grocery-credit eligibility pathway. The listed Idaho residency and full-year facts yield a $155 refundable grocery credit. +us,scenario_053,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model restricted refundable state credits to child-related credits and failed to apply Idaho's resident grocery credit. The childless head qualifies for the full $155 grocery credit. +us,scenario_053,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model found the correct grocery-credit pathway but used $120 rather than the 2026 base amount of $155. With all 12 months qualifying, no proration reduces the $155 credit." +us,scenario_053,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model incorrectly imposed an income cutoff on Idaho's grocery credit at this household's wage level. No phase-out or reduction applies, and the resident receives $155." +us,scenario_053,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,The model correctly found the resident eligible but used a $120 grocery-credit amount instead of the 2026 $155 base. Income does not reduce the full $155 credit. +us,scenario_053,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The zero answer omitted Idaho's refundable grocery credit. One qualifying resident for all 12 months receives $155. +us,scenario_053,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model correctly identified the refundable grocery credit but used a $100 amount. Idaho's applicable 2026 nonelderly base is $155. +us,scenario_053,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The zero answer failed to apply Idaho's grocery-credit pathway. Full-year qualification produces $155 in state refundable credits. +us,scenario_053,state_refundable_credits,inkling,llm_error,thresholds_rates,False,The model identified Idaho's grocery credit but substituted an approximate $120 amount for the exact 2026 parameter. The full-year credit is $155. +us,scenario_053,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for state_refundable_credits, violating the required output contract." +us,scenario_053,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model correctly applied full-year grocery-credit eligibility and the absence of monthly reductions but used $120 rather than the 2026 $155 base. Those same facts yield the full $155. +us,scenario_053,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model focused on child and property-tax conditions and omitted the separate Idaho grocery credit, which does not require either. The resident qualifies for $155 for the full year." +us,scenario_053,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model limited its analysis to state EITC and child-tax-credit pathways and overlooked Idaho's grocery credit. A childless full-year resident receives the $155 grocery credit. +us,scenario_053,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The zero answer omitted Idaho's refundable grocery credit for residents. This household qualifies for all 12 months, yielding $155." +us,scenario_054,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model applied only a CHIP upper-income threshold and treated income below that threshold as sufficient for eligibility. It omitted the required Medicaid-ineligibility test: Child 1 qualifies for Medicaid under the OLDER_CHILD category and therefore is not eligible for CHIP. +us,scenario_054,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model concluded that low income and lack of other coverage establish CHIP eligibility without checking Medicaid eligibility. Child 1's low income instead establishes Medicaid eligibility under the OLDER_CHILD category, which bars CHIP eligibility." +us,scenario_054,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated being below the CHIP income ceiling and lacking employer-sponsored insurance as sufficient. It omitted that CHIP covers children who do not qualify for Medicaid, while Child 1 qualifies for Medicaid under the OLDER_CHILD category." +us,scenario_054,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model conflated the Medicaid and CHIP income ranges, expressly counting income below the Medicaid threshold as CHIP-eligible. At this income, the age-10 child qualifies for Medicaid under the OLDER_CHILD category, so the Medicaid-exclusion rule makes CHIP eligibility false." +us,scenario_054,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model combined Medicaid and CHIP into a single coverage threshold and labeled qualification for either program as CHIP eligibility. Child 1 specifically qualifies for Medicaid under the OLDER_CHILD category, and that qualification excludes CHIP." +us,scenario_054,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stated the controlling condition—CHIP applies only if the child is not eligible for Medicaid—but never evaluated it. The engine's age-and-income test places Child 1 in the Medicaid OLDER_CHILD category, so the model's own stated condition yields no CHIP eligibility." +us,scenario_054,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to place Child 1 in North Carolina’s OLDER_CHILD Medicaid category and incorrectly treated unlisted citizenship, coverage, and household qualifiers as required negative evidence. The child is a dependent age 10 with MAGI at 0.58 times FPL, which satisfies the applicable child Medicaid pathway." +us,scenario_054,child1_wic_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model invented a WIC pathway for nutritionally at-risk children ages 6–18. WIC child eligibility ends at age five, so the 10-year-old is categorically ineligible regardless of household income or the head’s WIC receipt." +us,scenario_054,child1_wic_eligible,qwen3.8-max,llm_error,age_disability,False,"The model incorrectly treated age 10 as within WIC’s child eligibility range. Children qualify only while under age five, so Child 1 fails the categorical age requirement before the income test is relevant." +us,scenario_054,child2_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated income below the CHIP ceiling as sufficient for CHIP eligibility and failed to apply the Medicaid-exclusion step. Child 2 qualifies for Medicaid as a young child, which makes the child ineligible for CHIP regardless of employer-sponsored insurance." +us,scenario_054,child2_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated the combined Medicaid/CHIP income framework with CHIP eligibility and applied a CHIP income limit without first testing Medicaid eligibility. Child 2 qualifies for Medicaid under the young-child category, so CHIP eligibility is excluded before employer-sponsored insurance matters." +us,scenario_054,child2_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stated the correct conditional—CHIP only if the child is not Medicaid-eligible—but never evaluated it. Child 2 is Medicaid-eligible as a young child, so that condition fails and CHIP eligibility is 0." +us,scenario_054,child2_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model evaluated CHIP income limits and affordability of employer-sponsored insurance while omitting the prior Medicaid-eligibility exclusion. Child 2 qualifies for Medicaid in the young-child category, which categorically prevents CHIP eligibility; ESI affordability is irrelevant to this result." +us,scenario_054,child2_early_head_start_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated the $17,100 self-employment input as the program’s final countable household income and improperly relied on $14,000 of bank assets as evidence against eligibility. PolicyEngine’s Early Head Start income computation yields approximately zero income, below 100% of the federal poverty line, and the age-2 child therefore satisfies both the income and age requirements." +us,scenario_054,child2_head_start_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model applied the income test but omitted the minimum-age requirement for preschool Head Start. At age 2, Child 2 belongs to the Early Head Start age group and is not eligible for the requested Head Start output." +us,scenario_054,child2_head_start_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly placed age 2 within the preschool Head Start age range. Preschool Head Start begins at age 3, so Child 2 fails the age test." +us,scenario_054,child2_head_start_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The model treated low household income and age 2 as sufficient but skipped the preschool Head Start minimum age. Income eligibility cannot overcome Child 2's failure to be at least age 3. +us,scenario_054,child2_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,The model incorrectly treated active employer-sponsored insurance as a Medicaid eligibility bar. The child qualifies through North Carolina's YOUNG_CHILD category at 0.58 times FPL regardless of the listed employer-sponsored coverage. +us,scenario_054,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply the age-based YOUNG_CHILD Medicaid category and incorrectly demanded additional categorical indicators. At age 2 and 0.58 times FPL, Child 2 satisfies North Carolina's young-child income pathway; employer-sponsored insurance does not negate eligibility." +us,scenario_054,child2_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required explicit WIC receipt or an explicit statement of eligibility for Child 2. The requested output concerns eligibility rather than enrollment; age 2 satisfies WIC's categorical age rule, and the household meets the income test, yielding eligible." us,scenario_054,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_054,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model reported only its understated $3,995 EITC estimate despite also calculating a refundable CTC, so it omitted the CTC from the submitted total. It also failed to apply the 40% two-child EITC phase-in to adjusted net self-employment earnings, which yields $6,356.77." -us,scenario_054,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The reasoning computed a combined estimate but submitted an EITC-only figure of $6,269, violating its own aggregation. It also added the half-SE-tax deduction to the earned-income adjustment, understating the refundable CTC; the correct components are $6,356.77 of EITC and $2,008.79 of refundable CTC." -us,scenario_054,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly estimated the two-child EITC at roughly $3,000 even though the 40% phase-in applied to $15,891.93 yields $6,356.77. Its refundable CTC calculation also used $15,793 instead of the adjusted $15,891.93 earnings base, understating that component." -us,scenario_054,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model wrongly placed the household in the EITC phase-out region and reduced the EITC to about $4,700. At this income the two-child credit remains in its 40% phase-in, yielding $6,356.77, while the refundable CTC is $2,008.79 rather than roughly $2,200." -us,scenario_054,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model understated both components: the EITC is $6,356.77 rather than $6,164, and the refundable CTC is $2,008.79 rather than $1,914. It used inaccurate indexed parameters and an incorrect earnings base instead of applying the two-child phase-in and $2,500 CTC threshold to $15,891.93." -us,scenario_054,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The EITC calculation was correct, but the model used a $3,000 refundable-CTC earnings threshold. The applicable threshold is $2,500, so refundable CTC is $2,008.79 rather than $1,933.79." -us,scenario_054,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced earned income to $14,584 by subtracting the half-SE-tax deduction from an earnings figure already adjusted for self-employment tax. The credit calculations use $15,891.93, producing $6,356.77 of EITC and $2,008.79 of refundable CTC; it also used the obsolete $3,000 CTC threshold." -us,scenario_054,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"Its total implies only about $497 of refundable CTC after an approximately correct EITC estimate. Applying the refundable-CTC formula to $15,891.93 of adjusted net self-employment earnings yields $2,008.79, which must be added to the $6,356.77 EITC." -us,scenario_054,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly derived the EITC but applied a $3,000 threshold to the refundable CTC. Using the applicable $2,500 threshold raises refundable CTC from $1,933.79 to $2,008.79." -us,scenario_054,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an obsolete $3,000 refundable-CTC earnings threshold and an obsolete $1,000-per-child cap. The applicable $2,500 threshold applied to adjusted net earnings produces refundable CTC of $2,008.79, alongside the $6,356.77 EITC." -us,scenario_054,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted total matches the shortcut of adding the correct $6,356.77 EITC to refundable CTC calculated with a $3,000 earnings threshold. The applicable $2,500 threshold yields $2,008.79 of refundable CTC and a total of $8,365.56." -us,scenario_054,federal_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model omitted the EITC entirely even though self-employment income is earned income and both children qualify for the two-child EITC. It also calculated refundable CTC from gross $17,100 rather than adjusted net earnings; the correct components are $6,356.77 and $2,008.79." -us,scenario_054,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model denied all refundable credits despite $17,100 of self-employment income and two qualifying children. Adjusted net self-employment earnings generate a $6,356.77 two-child EITC and $2,008.79 refundable CTC." -us,scenario_054,federal_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The prompt supplies the filing assumption, household composition, earned self-employment income, and two qualifying children needed to calculate both credits; no additional explicit eligibility indicators are required. Ignoring those facts eliminated the $6,356.77 EITC and $2,008.79 refundable CTC." -us,scenario_054,federal_refundable_credits,gpt-5.5,llm_error,payroll_tax_base,False,"The model applied both credit formulas directly to gross self-employment income of $17,100. Refundable-credit earned income is the adjusted net amount of $15,891.93, producing EITC of $6,356.77 and refundable CTC of $2,008.79." -us,scenario_054,federal_refundable_credits,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model used gross self-employment income as the earned-income base, yielding $6,840 of EITC and $2,190 of refundable CTC. Applying the formulas to adjusted net self-employment earnings of $15,891.93 yields $6,356.77 and $2,008.79." -us,scenario_054,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The stated $5,834 EITC is consistent with an erroneously reduced earnings base near $14,585 rather than $15,891.93. That same base and an incorrect CTC threshold understated refundable CTC to $1,813 instead of $2,008.79." -us,scenario_054,federal_refundable_credits,grok-4.3,llm_error,other,False,"The model did not apply the two-child EITC and refundable-CTC formulas to the household's adjusted net self-employment earnings. Those calculations yield $6,356.77 and $2,008.79 respectively, already exceeding its unexplained $5,978 combined estimate." -us,scenario_054,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used the obsolete $3,000 ACTC earnings threshold, reducing refundable CTC to about $1,919. The applicable threshold is $2,500, which yields $2,008.79; its small EITC discrepancy also comes from rounding adjusted net earnings rather than using $15,891.93." -us,scenario_054,federal_refundable_credits,grok-build-0.1,llm_error,payroll_tax_base,False,"The model applied the EITC to gross $17,100 rather than adjusted net self-employment earnings and imposed obsolete post-reversion CTC parameters. The correct bases and parameters yield $6,356.77 of EITC and $2,008.79 of refundable CTC." -us,scenario_054,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for federal_refundable_credits. -us,scenario_054,federal_refundable_credits,minimax-m3,llm_error,other,False,"The submitted $672 contradicts the model's own stated $7,932 combined estimate and does not represent either refundable-credit component. It also used an erroneous $13,093 earnings base and treated the per-child refundable CTC cap as automatically payable instead of applying the earned-income formula." -us,scenario_054,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly treated SSTB self-employment income as excluded from earned income for EITC and refundable CTC purposes; SSTB status concerns the qualified business income deduction, not these credits. The income generates a $6,356.77 EITC and $2,008.79 refundable CTC even with zero regular tax liability." -us,scenario_054,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required an explicit free-meal indicator or stated SNAP amount instead of calculating eligibility from the household facts. It missed both independent free-tier pathways: income at 63% of the federal poverty guideline is below 130%, and the household's calculated positive SNAP benefit establishes categorical eligibility." -us,scenario_054,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model misclassified the head as a non-disabled childless adult and evaluated the wrong adult Medicaid category. The household includes two children, so the 40-year-old head is in North Carolina's parent/caretaker category, and MAGI of 0.58 FPL satisfies that pathway." -us,scenario_054,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to use the dependent-child household composition as the categorical Medicaid indicator for a parent/caretaker. It treated pregnancy, disability, or another special status as required, but PolicyEngine places the head in the parent/caretaker category and applies the MAGI income test at 0.58 FPL." -us,scenario_054,head_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model applied an incorrect North Carolina Medicaid income threshold for the head's category. PolicyEngine computes the head's MAGI as 0.58 FPL under the parent/caretaker pathway, which is below the applicable limit, not above it." -us,scenario_054,head_medicaid_eligible,minimax-m3,llm_error,other,False,"The model's explanation concludes the head should be eligible because income is below the 138% FPL adult threshold, but it submitted value = 0. This is a response-contract failure: its numeric output contradicts its own final eligibility reasoning, and it also failed to anchor eligibility in the parent/caretaker category used by PolicyEngine." -us,scenario_054,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model recognized the head is a parent but then discarded the parent/caretaker Medicaid pathway by treating North Carolina as if only traditional elderly, disabled, pregnant, or very narrow adult categories applied. PolicyEngine classifies the head as PARENT and uses MAGI of 0.58 FPL, which qualifies under the North Carolina parent/caretaker rule." -us,scenario_054,head_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model transferred Child 2’s under-five categorical eligibility to the head, but WIC does not qualify an adult merely because an eligible child lives in the household. It also treated reported receipt and an irrelevant asset test as proof of eligibility despite the head having no pregnant, breastfeeding, or postpartum status." -us,scenario_054,head_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented a qualifying-caretaker pathway for WIC. A 40-year-old caretaker does not qualify through the presence of a child under five; the head must independently be pregnant, breastfeeding, or postpartum." -us,scenario_054,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly listed WIC’s participant categories but then substituted the child’s age and the head’s reported receipt for the head’s own categorical status. The head has no pregnant, breastfeeding, or postpartum status, so passing the income test does not establish eligibility." -us,scenario_054,head_wic_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model inferred an unlisted postpartum or breastfeeding status from reported WIC receipt, contrary to the instruction that unlisted statuses are false. Receipt is not itself a categorical eligibility pathway, and the head therefore fails WIC’s adult-category test." -us,scenario_054,head_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model created a caretaker or family-member category that WIC does not provide. The young child can qualify individually, but the head cannot qualify without being pregnant, breastfeeding, or postpartum; low income and current receipt do not cure that failure." -us,scenario_054,head_wic_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model equated reported WIC receipt with eligibility and skipped the person-level categorical test. The head is not pregnant, breastfeeding, postpartum, an infant, or a child under five, so the head is ineligible." -us,scenario_054,head_wic_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated benefit receipt as conclusive evidence of eligibility instead of applying WIC’s person-specific categories. The head lacks every qualifying adult status, while the age-two child’s eligibility does not extend to the head." -us,scenario_054,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model acknowledged that pregnancy or postpartum status was not stated but nevertheless overrode the prompt’s false-by-default rule using reported receipt and the presence of a young child. Neither fact supplies the head with the required pregnant, breastfeeding, or postpartum status." -us,scenario_054,head_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model conflated the input indicating WIC receipt with the requested PolicyEngine eligibility output. Eligibility remains person-specific, and the head fails the adult categorical requirement because pregnancy, breastfeeding, and postpartum statuses are all false." -us,scenario_054,head_wic_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model used reported participation as the entire eligibility determination. The head’s receipt does not replace WIC’s categorical test, which the head fails without pregnant, breastfeeding, or postpartum status." -us,scenario_054,head_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model assumed that existing receipt confirms current eligibility and omitted the categorical assessment. The head is a 40-year-old with no qualifying pregnancy, breastfeeding, or postpartum status, and the eligible two-year-old does not confer eligibility on the head." -us,scenario_054,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that employee Social Security and Medicare tax on wages is zero and that self-employment income is excluded from payroll_tax, then overrode that result with a fabricated $1,084 residual. It treated self-employment income or an invented low-income NC payroll benchmark as part of employee-side payroll tax instead of leaving it to the separate self_employment_tax output." -us,scenario_054,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model failed to provide a parseable payroll_tax value. The requested output required a numeric annual employee-side payroll tax, and the derivation yields zero because the household has no wages subject to employee payroll tax and self-employment tax is reported separately." -us,scenario_054,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the 185% reduced-price ceiling as sufficient and failed to test the superior free-meals tier first. Categorical eligibility and income at 63% of the poverty guideline place the household in the FREE tier, which makes reduced-price eligibility false." -us,scenario_054,reduced_price_school_meals_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model reversed the threshold comparison by claiming roughly $15,000 was above a free-meals threshold it estimated at roughly $32,500. The household's traced ratio is 63% of the poverty guideline, below the 130% free threshold, and categorical eligibility independently places it in the FREE tier, so no reduced-price support is returned." -us,scenario_054,self_employment_tax,claude-haiku-4.5,llm_error,other,False,"The model identified the correct formula but miscomputed the component taxes on the $15,791.85 SECA base. It understated both the 12.4% Social Security component and the 2.9% Medicare component, then introduced an irrelevant half-SE-tax deduction concept inside the liability calculation even though that deduction affects income tax, not self_employment_tax." -us,scenario_054,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model used the correct formula, $17,100 × 0.9235 × 0.153, but performed the arithmetic or rounding incorrectly. That product is $2,416.15, not $2,415.00." -us,scenario_054,self_employment_tax,gpt-5.4-mini,llm_error,other,False,"The model used the right SECA structure but replaced the exact computation with a rough estimate. Applying 15.3% to 92.35% of $17,100 gives $2,416.15, so the $2,413 answer reflects imprecise arithmetic rather than a different rule." -us,scenario_054,self_employment_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct formula and even recognized it gives about $2,414 to $2,416, then submitted $1,596 with no valid SECA adjustment leading to that amount. Excluding Additional Medicare Tax does not reduce the basic 12.4% Social Security plus 2.9% Medicare tax, so the correct liability remains $2,416.15." -us,scenario_054,self_employment_tax,minimax-m3,llm_error,other,False,"The model computed the correct SECA base and nearly the correct tax in its explanation, but then submitted $2,419.95 instead of the computed product. The correct multiplication is $15,791.85 × 0.153 = $2,416.15, and no rounding convention turns that into $2,419.95." -us,scenario_054,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model projected an $820 maximum monthly allotment instead of applying the actual 2026 monthly maxima of $785–$803. Its net-income calculation was close, but the inflated allotment produced $540 per month instead of approximately $505–$525." -us,scenario_054,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied a conventional SNAP asset limit to the $14,000 bank balance. This household passes North Carolina categorical eligibility through TANF non-cash assistance, so that balance does not bar SNAP eligibility." -us,scenario_054,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used speculative COLA adjustments after already deriving a benefit near $6,456, then replaced that calculation with $7,152. The applicable monthly maxima are $785–$803, and summing the resulting monthly benefits yields $6,125.69." -us,scenario_054,snap,claude-opus-4.8,llm_error,other,False,"The model correctly derived a monthly benefit near $508 and an annual amount near $6,096, then discarded that result and asserted an unexplained $707 monthly benefit. Nothing in its stated deductions supports the submitted $8,484." -us,scenario_054,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model estimated $583 per month even though its stated $800 maximum less 30% of roughly $931 net income is about $521, not $583. Applying the actual monthly maxima and aggregating all twelve months yields $6,125.69." -us,scenario_054,snap,claude-sonnet-4.6,llm_error,other,False,"The model's reasoning repeatedly concluded $7,548, while its submitted value was $5,532. This violates the required contract that the explanation's final amount match the numeric output." -us,scenario_054,snap,claude-sonnet-5,llm_error,other,False,"The model stated that roughly $768 minus a $280 contribution produced a $350 benefit, but that subtraction equals about $488. Its $4,200 annual answer therefore contradicts its own inputs and arithmetic." -us,scenario_054,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model fixed the maximum allotment at $781 for every month. PolicyEngine applies the actual 2026 monthly maxima of $785–$803 and sums the twelve monthly results, producing $6,125.69." -us,scenario_054,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a stale $766 maximum monthly allotment. The applicable 2026 maxima are $785–$803, so subtracting the expected contribution month by month yields a larger annual benefit of $6,125.69." -us,scenario_054,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"The submitted $3,720 implies only $310 per month and provides no deduction or allotment calculation supporting that reduction. The traced calculation uses roughly $931 of net monthly income and monthly benefits of approximately $505–$525." -us,scenario_054,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's $5,800.80 is consistent with using a stale maximum allotment near $766 throughout the year. The actual 2026 monthly maxima range from $785 to $803 and must be applied for their respective months." -us,scenario_054,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a $747 maximum allotment, below every applicable 2026 monthly maximum of $785–$803. That stale threshold depressed its monthly benefit to $464 instead of approximately $505–$525." -us,scenario_054,snap,gemini-3.6-flash,llm_error,other,False,"The answer provides no computation supporting $7,548. That amount overstates the traced result obtained from roughly $931 of net monthly income and the applicable $785–$803 monthly maxima." -us,scenario_054,snap,glm-5.2,llm_error,thresholds_rates,False,"The model approximated the annual maximum allotment as $9,600 instead of applying each 2026 monthly maximum. The exact monthly maxima and expected contributions sum to $6,125.69, not $6,317." -us,scenario_054,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model characterized countable income as very low and awarded a near-maximum benefit. SNAP net income remains about $931 per month after the earned-income and standard deductions, creating an expected contribution near $279 each month." -us,scenario_054,snap,gpt-5.4-nano,llm_error,missing_output,False,"The model ignored facts explicitly supplying a three-person household and $17,100 of annual self-employment income. Those inputs establish gross monthly income of $1,425 and support a positive SNAP calculation." -us,scenario_054,snap,gpt-5.5,llm_error,other,False,"The model correctly identified net income of $931 but subtracted 30% incorrectly: $785 minus about $279 equals about $506, not $575. It also failed to apply the later $803 monthly maximum when annualizing." -us,scenario_054,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model treated the $14,000 bank balance as disqualifying. North Carolina categorical eligibility through TANF non-cash assistance removes that resource barrier for this household." -us,scenario_054,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model correctly derived about $506 using the $785 maximum but multiplied that single monthly amount by twelve. The maximum rises to $803 for part of 2026, so the twelve distinct monthly benefits total $6,125.69." -us,scenario_054,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model applied the $785 maximum to every month and annualized $506 uniformly. PolicyEngine applies monthly maxima ranging from $785 to $803, so the year totals $6,125.69." -us,scenario_054,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model declared both income and assets excessive, although $1,425 of gross monthly income is only about 63–64% of the applicable poverty guideline. It also omitted North Carolina categorical eligibility through TANF non-cash assistance, under which the bank balance does not disqualify the household." -us,scenario_054,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used an annual maximum allotment of $9,192, equivalent to a stale $766 monthly maximum. The applicable 2026 monthly maxima are $785–$803, raising the annual result to $6,125.69." -us,scenario_054,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an estimated $811 maximum monthly allotment, exceeding the applicable $785–$803 range. This inflated the monthly benefit to about $531 and the annual estimate to $6,370." -us,scenario_054,snap,kimi-k2.6,llm_error,asset_resource,False,"The model imposed the ordinary countable-resource limit on the $14,000 bank balance. The household instead passes categorical eligibility through TANF non-cash assistance, so the balance does not cause ineligibility." -us,scenario_054,snap,kimi-k3,llm_error,period_annualization,False,"The model used a fixed $783 maximum and a rounded $280 contribution for every month. The applicable maximum is $785–$803 across 2026, and the exact twelve monthly calculations sum to $6,125.69." -us,scenario_054,snap,minimax-m3,llm_error,thresholds_rates,False,"The model compared $17,100 to an annual 130%-of-poverty figure near $26,000 and then reversed the comparison: $17,100 is below $26,000. The household also passes categorical eligibility and receives a positive allotment." -us,scenario_054,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model awarded the maximum benefit despite acknowledging earned income and applicable deductions. Net income remains about $931 per month, so SNAP requires an expected household contribution near $279 rather than a zero contribution; it also used an inflated $934 maximum instead of $785–$803." -us,scenario_054,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model correctly computed that federal AGI after the half self-employment-tax deduction was about $15,892 and that the North Carolina head-of-household standard deduction of $19,125 reduced NC taxable income to zero. It then submitted $245 anyway, a parse-contract/substantive inconsistency that applies a positive NC tax despite its own taxable-income calculation being floored at zero." -us,scenario_054,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model abandoned the head-of-household filing status implied by a head with dependents and switched to a single-filer North Carolina standard deduction, then further replaced that calculation with an unsupported $596 estimate. Under the applicable head-of-household deduction, the roughly $15,892 AGI is fully offset and NC taxable income is zero." -us,scenario_054,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model applied North Carolina's flat tax rate to a positive taxable-income base without carrying through the standard-deduction calculation that eliminates taxable income. The correct NC computation floors taxable income at zero after the applicable deduction, so no pre-refundable-credit state income tax remains." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model stopped at the $18,150 standard deduction plus the $1,123.45 senior deduction, leaving taxable income at $137,003. It omitted the remaining $1,000 deduction included in PolicyEngine's $20,273.45 total, so it taxed $1,000 too much at 24%." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The submitted $14,829 contradicts the model's own bracket calculation, which first produced tax above $25,000. It then asserted an unexplained $10,000-plus reduction after adding taxable Social Security, even though adding $4,880.71 of taxable income increases rather than decreases tax." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model treated the $6,000 senior deduction as phaseout-free despite AGI of $156,275.86; PolicyEngine phases it down to $1,123.45. Its stated bracket arithmetic produced $24,431, then it replaced that result with $13,802 without identifying any deduction or credit that could cause the reduction." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used an estimated $18,600 standard deduction and omitted the separately phased-down $1,123.45 senior deduction and the rest of PolicyEngine's $20,273.45 deduction total. This left taxable income $1,673.59 above the engine result and it also used estimated rather than actual 2026 bracket thresholds." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included personal auto-loan interest in itemized deductions and never computed the charitable-contribution limitation or the phased-down senior deduction. It then stated taxable income near $138,526 but produced $24,196, which does not follow from applying the ordinary single-filer brackets to that income." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a projected $17,700 age-enhanced standard deduction and omitted the $1,123.45 phased-down senior deduction and the remainder of PolicyEngine's $20,273.45 deduction total. Its taxable income was therefore $2,573.59 too high, and its projected bracket thresholds compounded the overstatement." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model computed bracket tax of $24,788 from its stated assumptions, then reduced the submitted result to $18,033 through an unexplained “reconciliation.” No nonrefundable credit applies, and the senior deduction is $1,123.45 rather than the model's inconsistent estimates of roughly $4,000 or $8,175." -us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed a return to pre-TCJA law, using a personal exemption and 10%, 15%, 25%, and 28% brackets. PolicyEngine applies the operative 2026 standard-deduction regime, the phased-down senior deduction, and the 10%, 12%, 22%, and 24% brackets relevant to this income." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA provisions expired, restored a $5,300 personal exemption, and applied pre-TCJA rates. The operative 2026 computation instead uses $20,273.45 of taxable-income deductions and the current 10%, 12%, 22%, and 24% bracket schedule." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model included all $5,742 of Social Security rather than the taxable $4,880.71 and used an unsupported $16,800 deduction. More decisively, $17,296 does not result from applying the progressive single-filer rates to its own stated taxable income of $140,337." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model itemized $16,484 and claimed a personal exemption under an expired-law framework. PolicyEngine instead applies total deductions of $20,273.45 and the operative 2026 brackets, yielding taxable income of $136,002.41." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model restored a $5,050 personal exemption and used post-expiration pre-TCJA brackets. Those rules do not govern PolicyEngine's 2026 calculation, which uses $20,273.45 in deductions and the current bracket structure." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model referenced only the standard deduction and age-65 allowance and did not apply PolicyEngine's full $20,273.45 deduction total, including the $1,123.45 phased-down senior deduction. Its resulting taxable income and $26,105.17 tax are therefore too high." -us,scenario_055,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model deducted only $17,450 and omitted $2,823.45 of PolicyEngine's taxable-income deductions, including the phased-down senior deduction. That left taxable income at $138,825.70 rather than $136,002.41, and it also relied on estimated bracket thresholds." -us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model invoked a nonexistent rule that the standard deduction is reduced by “excess itemized deductions.” Its $16,348 answer does not follow from the required $136,002.41 taxable income or from applying ordinary 2026 single-filer rates, and no nonrefundable credit supplies the missing reduction." -us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated deductions as offsetting essentially all of $156,275.86 of gross income. PolicyEngine deducts only $20,273.45, leaving $136,002.41 taxable and a substantial liability; the absence of child or refundable credits does not reduce pre-refundable-credit tax to zero." -us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,missing_output,False,"The model submitted a zero placeholder instead of performing the requested estimate. The supplied facts determine $156,275.86 of gross income, $20,273.45 of deductions, $136,002.41 of taxable income, and $25,238.58 of tax." -us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model stated only that it used a senior-enhanced standard deduction and supplied $24,379 without showing a derivation. The correct deduction total is $20,273.45 and the resulting $136,002.41 of taxable income produces $25,238.58, so its answer embodies an excessive deduction or an understated bracket calculation." -us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model included the full $5,742 Social Security benefit instead of the taxable $4,880.71 and therefore overstated gross income by $861.29. Despite that overstatement, it reported only $21,842 on taxable income it called $140,137, which is not the result of applying the 2026 progressive brackets and has no supporting credit." -us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed post-TCJA-expiration law, restored a personal exemption, and applied 10%, 15%, 25%, and 28% rates. PolicyEngine uses the operative 2026 deduction regime and current brackets, with taxable income of $136,002.41." -us,scenario_055,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an expired-law $9,650 standard deduction and pre-TCJA 10%, 15%, 25%, and 28% brackets. It also calculated taxable income as $139,792 without subtracting the personal exemption associated with the regime it invoked, making its derivation internally inconsistent." -us,scenario_055,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model deducted only a projected $17,400 standard deduction and omitted $2,873.45 of PolicyEngine's total deductions, including the $1,123.45 phased-down senior deduction. This produced taxable income near $138,876 instead of $136,002.41 and overstated the tax." -us,scenario_055,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model invented mortgage interest from the mortgage balance despite the instruction that unlisted amounts are zero, misstated AGI, and invoked unidentified nonrefundable credits to erase roughly $18,500 of tax. No such credits apply, so the liability cannot be reduced to zero." -us,scenario_055,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model calculated only $2,045 of taxable Social Security even though high provisional income makes $4,880.71 taxable. It also used the wrong deduction comparison, counted medical expenses despite the 7.5% AGI floor, and omitted the phased-down senior deduction, producing the wrong taxable-income base." -us,scenario_055,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model identified the correct 7.65% combined employee rate and the correct $83,635 wage base, but rounded the result to $6,400 instead of returning the engine-calculated $6,398.09." -us,scenario_055,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model stated the correct wage-only base and 6.2% plus 1.45% employee rates, but its $10,534 answer does not follow that computation. Those rates applied to $83,635 yield $6,398.09, not $10,534." -us,scenario_055,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model used the combined employer-and-employee rates of 12.4% for Social Security and 2.9% for Medicare even though the requested output includes only employee-side payroll tax. It then applied an unsupported reduction for allegedly nontaxable wages; all $83,635 of wages are subject to the employee rates, yielding $6,398.09." -us,scenario_055,payroll_tax,gpt-5.6-luna,llm_error,missing_output,False,"The model substituted a zero placeholder instead of calculating payroll tax from the supplied $83,635 of wages. The wage amount provides the required base, and the employee Social Security and Medicare components total $6,398.09." -us,scenario_056,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the listed alimony and self-employment income as producing positive taxable income after itemized deductions, then applied a small regular income tax. It missed that the HSA deduction and losses reduce AGI below zero and that the standard deduction leaves no taxable income, so federal income tax before refundable credits is $0 before any credits are considered." -us,scenario_056,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly excluded childless non-elderly adults from New Jersey Medicaid, overlooking the ACA adult expansion category. It also incorrectly treated employer-sponsored insurance as a Medicaid disqualification and used gross alimony plus self-employment receipts instead of the engine’s MAGI result of 0.00 times FPL." -us,scenario_056,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model added the $12,000 financial-assistance amount to MAGI and then contradicted its own arithmetic by calling $17,839 above a stated $21,597 threshold. The applicable computation places MAGI at 0.00 times FPL, and employer-sponsored insurance does not bar ACA-expansion Medicaid." -us,scenario_056,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted financial assistance, alimony, and gross self-employment income as $17,839 of Medicaid MAGI. PolicyEngine’s MAGI computation yields 0.00 times FPL, placing this 20-year-old adult below New Jersey’s ACA-expansion limit." -us,scenario_056,head_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model correctly recognized that the stated gross-income total was below its own estimate of the expansion threshold, but then incorrectly treated employer-sponsored insurance as disqualifying coverage. Medicaid eligibility under New Jersey’s ACA adult category is not barred by existing employer coverage." -us,scenario_056,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model acknowledged that the person is a low-income childless adult below the ACA expansion limit, then wrongly required a separate parent, pregnancy, disability, or elderly category. The ACA adult expansion category is itself the applicable categorical pathway, and employer-sponsored insurance does not defeat it." -us,scenario_056,head_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,The model applied a nonexistent rule that access to or enrollment in employer-sponsored insurance disqualifies a person from Medicaid. The head qualifies through New Jersey’s income-based ACA adult expansion pathway despite having employer coverage. -us,scenario_056,head_medicaid_eligible,gpt-5.4-mini,llm_error,health_coverage,False,"The model treated employer-sponsored insurance as evidence against Medicaid eligibility and failed to test the ACA adult expansion pathway. At age 20, with MAGI at 0.00 times FPL, the head satisfies New Jersey’s expansion-adult income rule." -us,scenario_056,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model wrongly required disability, pregnancy, or another traditional Medicaid category. New Jersey’s ACA expansion independently covers qualifying nondependent adults under 65, and this head’s MAGI is 0.00 times FPL." -us,scenario_056,head_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"The model incorrectly treated employer-sponsored insurance as disqualifying and asserted an income failure without computing Medicaid MAGI. The engine places MAGI at 0.00 times FPL, below New Jersey’s ACA adult expansion threshold." -us,scenario_056,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract." -us,scenario_056,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model asserted that income exceeded New Jersey’s Medicaid threshold without applying the Medicaid MAGI computation. PolicyEngine calculates MAGI at 0.00 times FPL, while employer-sponsored insurance does not disqualify the head from the ACA adult category." -us,scenario_056,head_medicaid_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model incorrectly treated employer-sponsored insurance as an absolute Medicaid bar and improperly summed financial assistance, alimony, and gross self-employment income into Medicaid MAGI. The applicable MAGI result is 0.00 times FPL, which satisfies New Jersey’s ACA adult expansion income test." +us,scenario_054,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model computed an EITC of only $3,995 instead of applying the 40% two-child phase-in to net self-employment earnings, which yields $6,356.77. It also omitted the $2,008.79 refundable CTC from its submitted total despite calculating a separate refundable CTC in its reasoning." +us,scenario_054,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model’s own recomputation produced an $8,686 combined estimate, but it submitted $6,269, essentially an EITC-only figure. The required total includes both the $6,356.77 EITC and $2,008.79 refundable CTC." +us,scenario_054,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model badly understated the EITC after correctly recognizing that the household was in the two-child phase-in range, assigning roughly $3,000 rather than $6,356.77. It also used an incorrect 45% phase-in rate; the applicable two-child EITC rate is 40%." +us,scenario_054,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly placed the household in the EITC phaseout region and reduced the EITC to about $4,700. Net self-employment earnings remain in the 40% phase-in range, producing a $6,356.77 EITC before adding the $2,008.79 refundable CTC." +us,scenario_054,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model understated both components, using a $6,164 EITC instead of $6,356.77 and a $1,914 refundable CTC instead of $2,008.79. The refundable CTC is 15% of qualifying earned income above $2,500, subject to the per-child cap, not a reduced interaction amount." +us,scenario_054,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model correctly computed the $6,356.77 EITC but applied a $3,000 refundable-CTC earned-income threshold. Using the applicable $2,500 threshold raises the refundable CTC from $1,933.79 to $2,008.79." +us,scenario_054,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced qualifying earned income to $14,584 by subtracting the half-self-employment-tax deduction again when calculating the credits. The credit calculation uses net earnings from self-employment of about $15,891.93, yielding a $6,356.77 EITC and $2,008.79 refundable CTC." +us,scenario_054,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The submitted total leaves only $497 for the refundable CTC after the stated $6,370 EITC. The refundable CTC is $2,008.79, and the precisely calculated EITC is $6,356.77." +us,scenario_054,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model correctly computed the $6,356.77 EITC but used a $3,000 threshold for the refundable CTC. The applicable $2,500 threshold produces $2,008.79 rather than $1,933.79." +us,scenario_054,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model used an obsolete $3,000 earned-income threshold and an incorrect $1,000-per-child refundable cap for the CTC. Applying the $2,500 threshold produces a $2,008.79 refundable CTC, which combines with the $6,356.77 EITC." +us,scenario_054,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The submitted $8,291 total matches the shortcut that uses the correct EITC but a refundable CTC calculated above a $3,000 threshold. The applicable $2,500 threshold makes the refundable CTC $2,008.79 and the total $8,365.56." +us,scenario_054,federal_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model omitted the EITC entirely even though the head has qualifying self-employment earnings and two qualifying children. It also treated gross business income as credit earned income; the correct components are a $6,356.77 EITC and $2,008.79 refundable CTC." +us,scenario_054,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model denied positive refundable credits despite the explicitly assumed filing and the head’s qualifying self-employment earnings with two qualifying children. Those facts generate a $6,356.77 EITC and a $2,008.79 refundable CTC." +us,scenario_054,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required additional filing or eligibility indicators even though the prompt assumes filing and supplies the income, relationship, age, and household facts needed for both credits. The household qualifies for the $6,356.77 EITC and $2,008.79 refundable CTC." +us,scenario_054,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model applied the EITC and refundable-CTC formulas directly to $17,100 of gross self-employment income. Both calculations use net earnings from self-employment of about $15,891.93, producing $6,356.77 and $2,008.79 respectively." +us,scenario_054,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model treated the full $17,100 of gross self-employment income as earned income for both refundable credits. Using net earnings from self-employment yields a $6,356.77 EITC and $2,008.79 refundable CTC rather than $6,840 and $2,190." +us,scenario_054,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used an understated earned-income base, resulting in a $5,834 EITC and $1,813 refundable CTC. Net earnings from the stated self-employment income are about $15,891.93, which produce $6,356.77 and $2,008.79." +us,scenario_054,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model failed to apply the two-child EITC schedule correctly and submitted a total below the $6,356.77 EITC alone. Both children qualify for the EITC and CTC, and the additional $2,008.79 refundable CTC brings the total to $8,365.56." +us,scenario_054,federal_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model used the obsolete $3,000 earned-income threshold for the refundable CTC, understating that component as $1,919. The applicable $2,500 threshold yields $2,008.79; its EITC was also slightly understated instead of using the trace value of $6,356.77." +us,scenario_054,federal_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model applied the EITC to gross self-employment income and assumed a reverted $1,000-per-child refundable CTC cap. Net self-employment earnings produce a $6,356.77 EITC, while the applicable refundable CTC formula produces $2,008.79." +us,scenario_054,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model calculated the EITC from gross self-employment income rather than net earnings and treated the refundable CTC as a flat $1,000 per child. The correct calculations yield $6,356.77 of EITC and $2,008.79 of refundable CTC." +us,scenario_054,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, violating the required output contract." +us,scenario_054,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model submitted $672 even though its own reasoning calculated a $7,932 total, so the final value does not follow from its stated computation. Its reasoning also used an incorrect earned-income amount and incorrectly granted the full per-child refundable CTC cap instead of applying the 15% earned-income limitation." +us,scenario_054,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly excluded SSTB self-employment income from earned income for EITC and refundable-CTC purposes. SSTB classification concerns the qualified-business-income deduction, not whether net self-employment earnings support these refundable credits, and zero tax liability does not bar the refundable CTC." +us,scenario_054,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly limited the refundable CTC to the $375 left after an asserted offset against regular tax. With zero pre-refundable federal income tax, the earned-income formula supplies a $2,008.79 refundable CTC, and the two-child EITC is $6,356.77 rather than $5,757." +us,scenario_054,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required an explicit free-meal indicator or a stated SNAP amount instead of calculating eligibility from the household facts. Income at 63% of the federal poverty guideline independently satisfies the free-meal threshold of 130%, and SNAP participation also establishes categorical eligibility." +us,scenario_054,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated the head as a childless adult even though the household includes the head’s two children. It therefore omitted the parent/caretaker Medicaid pathway, under which the head’s MAGI of 0.58 times FPL qualifies." +us,scenario_054,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model searched only for pregnancy, disability, and similar indicators and failed to recognize that living with two dependent children places the head in the parent/caretaker category. That category and the head’s MAGI of 0.58 times FPL establish eligibility without an asset test." +us,scenario_054,head_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly placed the head above North Carolina’s applicable Medicaid income threshold. PolicyEngine computes MAGI at 0.58 times FPL and applies the parent/caretaker threshold, which the head satisfies." +us,scenario_054,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model’s own reasoning concluded that the head’s income was below the cited Medicaid limit and explicitly selected 1, but it submitted 0. It failed to carry its eligibility conclusion into the required numeric output; PolicyEngine classifies the head as a parent/caretaker at 0.58 times FPL." +us,scenario_054,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model wrongly treated the head as an adult without a qualifying dependent-child category and asserted that North Carolina had not expanded Medicaid. The head is a parent/caretaker, and PolicyEngine’s MAGI calculation of 0.58 times FPL satisfies the applicable eligibility pathway." +us,scenario_054,head_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly transferred Child 2's under-five categorical eligibility to the head. WIC has no caretaker pathway, and neither low income, assets, nor reported receipt supplies the head with the required pregnant, postpartum, or breastfeeding status." +us,scenario_054,head_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented a qualifying-caretaker category based on the presence of a two-year-old. WIC eligibility is person-specific, so a 40-year-old head without pregnancy, postpartum, or breastfeeding status is not eligible." +us,scenario_054,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly listed WIC's categorical groups but then treated the child's age and the head's reported receipt as satisfying the head's categorical test. The head belongs to none of the listed groups because pregnancy, postpartum, and breastfeeding are all unlisted and therefore false." +us,scenario_054,head_wic_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model inferred an unlisted postpartum or breastfeeding status from reported WIC receipt. The prompt requires unlisted statuses to be false, and income below 185% FPL does not replace WIC's categorical requirement." +us,scenario_054,head_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model fabricated a caretaker or family-member eligibility pathway from Child 2's age. WIC evaluates the head individually, and neither having an eligible child nor already receiving WIC establishes pregnancy, postpartum, or breastfeeding status." +us,scenario_054,head_wic_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model equated current WIC receipt with individual eligibility and skipped the categorical test. The 40-year-old head has no stated pregnancy, postpartum, or breastfeeding status, so the head fails that test." +us,scenario_054,head_wic_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated benefit receipt as conclusive proof of eligibility instead of applying person-level WIC rules. Reported receipt does not give the head a qualifying pregnant, postpartum, or breastfeeding status." +us,scenario_054,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model acknowledged that pregnancy and postpartum status were not stated but overrode the prompt's instruction that unlisted statuses are false. A young child and the head's reported receipt do not satisfy the adult head's categorical requirement. +us,scenario_054,head_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model directly equated receiving WIC with being eligible for WIC. Eligibility is person-specific, and the head lacks every qualifying adult categorical status." +us,scenario_054,head_wic_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model used stated WIC receipt as the eligibility rule and performed no categorical assessment. The head is age 40 and has no stated pregnancy, postpartum, or breastfeeding status." +us,scenario_054,head_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model assumed enrollment confirms eligibility. Reported receipt does not establish a qualifying WIC category for the head, whose pregnancy, postpartum, and breastfeeding statuses are all false under the prompt." +us,scenario_054,head_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented pregnancy, postpartum, or breastfeeding status despite none being listed. Income eligibility alone is insufficient, so the head fails WIC's categorical requirement." +us,scenario_054,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that the household has no wages and that self-employment income is excluded from employee payroll tax, but then replaced the resulting zero with an invented $1,084 residual benchmark. No such residual employee payroll tax applies in North Carolina, so the stated computation yields $0." +us,scenario_054,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the required payroll_tax output entirely. The derivation requires $0 because the household has no wages subject to employee Social Security, Medicare, Additional Medicare, or mandatory state employee payroll taxes." +us,scenario_054,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the 185% reduced-price ceiling as sufficient and failed to evaluate the superior free-meals tier. At 63% of the federal poverty guideline and with SNAP/TANF categorical eligibility, the household receives free meals, so reduced-price eligibility is 0." +us,scenario_054,reduced_price_school_meals_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model reversed the income comparison by asserting that roughly $15,000 was above a free-meals threshold it estimated at roughly $32,500. The household's traced ratio is 63% of the federal poverty guideline, below the 130% free threshold, and SNAP/TANF categorical eligibility independently places it in the FREE tier, leaving no reduced-price support." +us,scenario_054,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model made multiple arithmetic errors after correctly identifying the 92.35% net-earnings adjustment: $15,791.85 × 15.3% equals $2,416.15305, not $2,418.16 or $2,414.29. It also incorrectly invoked a deduction for half of the Social Security portion in computing the tax liability; the one-half-SE-tax deduction applies to adjusted gross income, not SECA tax." +us,scenario_054,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model identified the correct formula but rounded inaccurately. Applying it without premature approximation gives $17,100 × 0.9235 × 0.153 = $2,416.15305, which rounds to $2,416.15 rather than $2,415.00." +us,scenario_054,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model replaced the exact SECA computation with an inaccurate rough estimate. The stated formula yields $17,100 × 0.9235 × 0.153 = $2,416.15305, rounded to $2,416.15, not $2,413.00." +us,scenario_054,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's own formula produces approximately $2,416, but it submitted $1,596 without a valid computation connecting the two amounts. Excluding Additional Medicare Tax does not reduce the ordinary 12.4% Social Security plus 2.9% Medicare SECA liability, which equals $2,416.15." +us,scenario_054,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly described a result near $2,416.18 but submitted the inconsistent amount $2,419.95. Exact arithmetic using the unrounded base gives $15,791.85 × 15.3% = $2,416.15305, rounded to $2,416.15." +us,scenario_054,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the deduction for one-half of self-employment tax from the SECA liability and then produced a final number inconsistent with its intermediate figures. That deduction reduces adjusted gross income for income-tax purposes; it does not reduce self-employment tax, which remains $17,100 × 92.35% × 15.3% = $2,416.15." +us,scenario_054,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model substituted a projected $820 monthly maximum allotment for the applicable 2026 monthly values of $785-$803. With net income near $931 and contributions near $279, the twelve monthly calculations sum to $6,125.69, not $6,480." +us,scenario_054,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model imposed a conventional SNAP asset limit on the $14,000 bank balance. North Carolina categorical eligibility through TANF non-cash assistance removes that resource-test barrier, so the household remains eligible." +us,scenario_054,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model improperly reduced SNAP self-employment income by half of self-employment tax and then abandoned its own calculated $6,456 estimate for an unsupported $7,152 COLA adjustment. The trace uses about $931 of net monthly income and the actual monthly maximum allotments, producing $6,125.69." +us,scenario_054,snap,claude-opus-4.8,llm_error,other,False,"The model correctly derived a benefit near $508 per month and about $6,096 annually, then replaced it with an unexplained $707 monthly figure. Nothing in the SNAP computation supports that refinement; the traced monthly allotments are about $505-$525." +us,scenario_054,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model overstated the monthly benefit as about $583 after acknowledging net income near the traced amount. Applying the applicable $785-$803 maximum allotments and a $278-$279 contribution yields about $505-$525 per month, totaling $6,125.69." +us,scenario_054,snap,claude-sonnet-4.6,llm_error,other,False,"The submitted value of $5,532 contradicts the model's repeated final calculation and explanation of $7,548. Independently, its reasoning also applied an unsupported 40% self-employment business-cost deduction and stale projected allotments instead of the traced net income and month-specific maxima." +us,scenario_054,snap,claude-sonnet-5,llm_error,other,False,"The model's stated components imply roughly $488 per month, not the submitted $350 per month. It introduced an unexplained reduction after calculating the maximum allotment minus a contribution near $280, causing the $4,200 result." +us,scenario_054,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a fixed $781 maximum allotment and a $202 standard deduction. The applicable maximum changes from $785 to $803 during the year and the traced net income is about $931, so summing the monthly results yields $6,125.69." +us,scenario_054,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the stale $766 maximum allotment for every month. The applicable 2026 maxima are $785-$803, which raise the monthly benefits from its $483.40 estimate to roughly $505-$525." +us,scenario_054,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"The unexplained $3,720 estimate implies only $310 per month, far below the traced $505-$525 monthly allotments. It omitted or overstated a core component of the maximum-allotment-minus-30%-of-net-income calculation." +us,scenario_054,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's $5,800.80 result uses an understated maximum allotment equivalent to the stale $766 figure. Applying the 2026 monthly maxima of $785-$803 to net income near $931 produces $6,125.69." +us,scenario_054,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a $747 maximum allotment, below every applicable 2026 monthly maximum of $785-$803. Its deduction calculation was close, but the obsolete allotment reduced the annual benefit to $5,573." +us,scenario_054,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The unexplained $7,548 answer corresponds to using an overstated benefit near $629 per month. The traced calculation subtracts a $278-$279 contribution from monthly maxima of $785-$803, producing only about $505-$525 per month." +us,scenario_054,snap,glm-5.2,llm_error,thresholds_rates,False,"The model assumed an annual maximum allotment of about $9,600 rather than aggregating the applicable monthly $785-$803 maxima. Its net-income calculation was close, but the overstated maximum pushed the result to $6,317." +us,scenario_054,snap,gpt-5.4-mini,llm_error,other,False,"The $8,868 estimate implies a $739 monthly benefit and therefore barely subtracts the household's expected contribution. SNAP requires subtracting 30% of net income, about $278-$279 monthly, leaving only $505-$525 per month." +us,scenario_054,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model falsely stated that income and household-size information were absent even though the prompt specifies three people and $17,100 of annual self-employment income. Those facts establish low income, and TANF non-cash categorical eligibility supplies the qualifying pathway." +us,scenario_054,snap,gpt-5.5,llm_error,other,False,"The model made a direct subtraction error: $785 minus 30% of $931 is about $506, not $575. It also failed to aggregate the changing monthly maxima, which together yield $6,125.69." +us,scenario_054,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model treated the $14,000 bank balance as disqualifying under a resource limit. TANF non-cash categorical eligibility makes the household eligible without that asset barrier." +us,scenario_054,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model correctly obtained about $506 using the $785 maximum but multiplied that single monthly result across the full year. The maximum rises as high as $803 in other months, so the twelve month-specific benefits total $6,125.69." +us,scenario_054,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model froze the maximum allotment at $785 for all twelve months, yielding $506 monthly and $6,072 annually. The applicable maximum varies from $785 to $803, and summing those monthly allotments produces $6,125.69." +us,scenario_054,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly declared both income and assets over the limits. Monthly gross income of $1,425 is far below the applicable poverty-based threshold, and TANF non-cash categorical eligibility prevents the $14,000 balance from disqualifying the household." +us,scenario_054,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used an annual maximum allotment of $9,192, equivalent to the stale $766 monthly maximum. The applicable 2026 maxima are $785-$803, so its otherwise similar net-income calculation understates the benefit." +us,scenario_054,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an estimated $811 monthly maximum, above the applicable $785-$803 range. That inflated the monthly benefit to about $531 instead of the traced $505-$525 amounts." +us,scenario_054,snap,inkling,llm_error,asset_resource,False,"The model imposed a North Carolina SNAP resource limit on the $14,000 bank balance. The household qualifies categorically through TANF non-cash assistance, so that balance does not eliminate SNAP eligibility." +us,scenario_054,snap,kimi-k2.6,llm_error,asset_resource,False,"The model applied the ordinary resource limit for a household without an elderly or disabled member. It missed the TANF non-cash categorical-eligibility pathway, under which the $14,000 bank balance does not disqualify this household." +us,scenario_054,snap,kimi-k3,llm_error,thresholds_rates,False,"The model correctly found $931 of net monthly income and correctly disregarded the bank balance, but used a fixed $783 maximum and rounded the contribution to $280. The engine applies monthly maxima of $785-$803 and contributions of about $278-$279, totaling $6,125.69." +us,scenario_054,snap,minimax-m3,llm_error,thresholds_rates,False,"The model compared $17,100 with an annual 130%-of-poverty figure near $26,000 and then reversed the inequality: $17,100 is below, not above, that threshold. The household also qualifies categorically through TANF non-cash assistance." +us,scenario_054,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model awarded the full maximum allotment despite acknowledging $17,100 of self-employment income. SNAP subtracts 30% of net income—about $278-$279 each month—from the $785-$803 maxima, so the household does not receive the maximum." +us,scenario_054,snap,qwen3.8-max,llm_error,period_annualization,False,"The model treated $1,579 as self-employment earnings even though the prompt gives $17,100 annually, or $1,425 monthly. At that monthly income, deductions produce net income near $931, which is below the eligibility limit and supports a positive benefit." +us,scenario_054,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly calculated North Carolina taxable income as $0 and tax as $0, then submitted $245 anyway. Its numeric output directly contradicts every computation step and conclusion in its explanation." +us,scenario_054,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model first derived $0 taxable income using the head-of-household standard deduction, then switched without basis to single filing status and ultimately submitted $596 without showing any additions or subtractions that create that liability. It discarded the zero tax base and invented a positive North Carolina tax unsupported by its own arithmetic." +us,scenario_054,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's $142 answer implies that it retained a positive North Carolina taxable-income base after the standard deduction. The applicable computation yields no positive taxable income, so applying the flat rate must produce $0 rather than $142." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model applied the charitable-contribution floor but then compared $15,703 of itemized deductions with the standard deduction alone, omitting the deductible auto-loan interest that raises the itemized base to $17,545.98. Adding the $1,123.45 senior deduction produces total deductions of $20,273.45 and taxable income of $136,002.41, not $137,003." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The submitted $14,829 does not follow from the model's own bracket calculation, which first produced tax around $25,392 and then replaced it with an unsupported figure. It also used the full Social Security benefit in gross income before inconsistently switching to an approximate taxable portion instead of using taxable Social Security of $4,880.71." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model treated the $6,000 senior deduction as phaseout-free even though income of $156,275.86 reduces it to $1,123.45. Its own intermediate tax was $24,431, and the later drop to $13,802 had no deduction or credit calculation supporting it." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted both the phased-down $1,123.45 senior deduction and the deductible auto-loan interest from the itemized-deduction comparison. The resulting deduction is $20,273.45 and taxable income is $136,002.41, rather than the model's $137,676." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model added auto-loan interest without applying the complete 2026 itemized-deduction rules and then used an approximate $17,750 deduction instead of the traced $20,273.45 total deduction. It also asserted a $24,196 bracket result that does not follow from its stated taxable income of roughly $138,526." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated personal auto-loan interest as wholly nondeductible and omitted the separate phased-down senior deduction of $1,123.45. Those omissions raised taxable income from $136,002.41 to $138,576 and consequently overstated the tax." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model first computed tax of $24,788 from its stated inputs, then reduced it to $18,033 through an invented reconciliation with no tax provision or credit. It also used an approximate $4,000 senior deduction instead of the income-phaseout result of $1,123.45." +us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed a return to pre-TCJA rules, using a personal exemption and 10%, 15%, 25%, and 28% brackets. The operative 2026 calculation has no personal exemption and applies the enacted 10%, 12%, 22%, and 24% schedule to $136,002.41." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration restored a personal exemption and the pre-TCJA 10%, 15%, 25%, and 28% rate schedule. The 2026 rules instead yield $136,002.41 of taxable income under the enacted 10%, 12%, 22%, and 24% brackets." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model used the full $5,742 Social Security benefit in gross income and supplied no bracket arithmetic capable of producing $17,296. Taxable Social Security is $4,880.71, total deductions are $20,273.45, and tax on the resulting $136,002.41 is $25,238.58." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model improperly subtracted a personal exemption and used an unspecified 2026 bracket calculation consistent with obsolete pre-TCJA rules. The applicable calculation uses no personal exemption, total deductions of $20,273.45, and the enacted brackets on $136,002.41." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly restored a $5,050 personal exemption and postulated post-TCJA-expiration rate rules. The operative 2026 law uses no personal exemption and taxes $136,002.41 under the 10%, 12%, 22%, and 24% brackets." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used only an unspecified standard deduction and age allowance, omitting the itemized-deduction comparison and the separate phased-down senior deduction. The correct deduction total is $20,273.45, leaving $136,002.41 taxable." +us,scenario_055,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model stopped at an estimated $17,450 standard deduction and omitted the deductible auto-loan-interest treatment and the $1,123.45 phased senior deduction. The traced deduction total is $20,273.45, reducing taxable income from its $138,825.70 estimate to $136,002.41." +us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model misstated the deduction mechanism by saying the standard deduction is reduced by excess itemized deductions; taxpayers instead deduct the greater qualifying base and then apply the separate senior deduction. Its $16,348 answer is unsupported by any taxable-income or bracket calculation, whereas $136,002.41 of taxable income yields $25,238.58." +us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated deductions and nonexistent nonrefundable-credit effects as offsetting nearly $156,276 of gross income. Total deductions are only $20,273.45, no nonrefundable credit reduces the liability, and $136,002.41 remains taxable." +us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,missing_output,False,"The model submitted a zero placeholder instead of performing the requested calculation. The supplied facts determine gross income of $156,275.86, deductions of $20,273.45, taxable income of $136,002.41, and tax of $25,238.58." +us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model invoked a generic senior-enhanced standard deduction but did not calculate the itemized alternative or the income-phased senior deduction. The applicable total deduction is $20,273.45, and taxing the resulting $136,002.41 produces $25,238.58 rather than $24,379." +us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model included the entire $5,742 Social Security benefit, producing gross income of $157,137 instead of taxable Social Security of $4,880.71 and gross income of $156,275.86. It then used an unspecified standard deduction and bracket calculation that does not support its $21,842 result." +us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed post-TCJA expiration, restored a personal exemption, and applied the obsolete 10%, 15%, 25%, and 28% brackets. The 2026 calculation uses no personal exemption and applies the enacted rate schedule to $136,002.41." +us,scenario_055,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly used a post-TCJA-expiration standard deduction and the obsolete 10%, 15%, 25%, and 28% brackets. It also failed to apply the $1,123.45 senior deduction, leaving taxable income overstated before applying the wrong rates." +us,scenario_055,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and used pre-TCJA rates for 2026. The operative rules provide total deductions of $20,273.45 without a personal exemption and tax $136,002.41 under the enacted brackets." +us,scenario_055,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model compared itemized deductions with an estimated standard deduction but omitted deductible auto-loan interest and the separate $1,123.45 phased senior deduction. This understated deductions at about $17,400 instead of $20,273.45 and overstated taxable income by roughly $2,874." +us,scenario_055,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model invented mortgage interest from the mortgage balance despite the instruction that unlisted numeric inputs are zero, misstated AGI, and then subtracted unspecified nonrefundable credits to erase an admitted tax near $18,500. No such credits apply, and the actual deduction and bracket calculation produces $25,238.58." +us,scenario_055,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model taxed only $2,045 of Social Security even though the high provisional income makes $4,880.71 taxable. It also omitted deductible auto-loan interest, incorrectly treated medical spending below the 7.5% floor as deductible, and failed to apply the $1,123.45 senior deduction." +us,scenario_055,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model supplied no derivation, and its $30,253 answer exceeds the tax generated by the applicable 2026 brackets on the traced taxable income. Gross income of $156,275.86 less $20,273.45 of deductions leaves $136,002.41 taxable and produces $25,238.58." +us,scenario_055,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model identified the correct employee rates and wage base but rounded the computed liability to $6,400 instead of returning the component-level result. Applying those rates to $83,635 yields $5,185.38 of Social Security tax plus $1,212.71 of Medicare tax, totaling $6,398.09." +us,scenario_055,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model’s submitted $10,534 contradicts its stated computation of 6.2% Social Security plus 1.45% Medicare on $83,635. Its own stated formula yields $6,398.09, so it made an arithmetic or final-answer transcription error." +us,scenario_055,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model used the combined employer-and-employee rates of 12.4% for Social Security and 2.9% for Medicare even though the requested output includes only employee-side payroll tax. It then invented a partial nontaxability adjustment unsupported by the facts instead of calculating the employee components of $5,185.38 and $1,212.71." +us,scenario_055,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model replaced the required calculation with a zero placeholder despite the explicitly listed $83,635 of annual wages. Those wages generate $5,185.38 of employee Social Security tax and $1,212.71 of employee Medicare tax, totaling $6,398.09." +us,scenario_056,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the listed alimony and self-employment amounts as producing positive taxable income and focused on itemized deductions, but failed to apply the $9,915 HSA deduction and the standard deduction after adjustments. Those steps leave no taxable income, so its $125 liability has no taxable-income base." +us,scenario_056,head_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model wrongly asserted that childless New Jersey adults are generally excluded and that employer-sponsored insurance disqualifies them. New Jersey’s ACA expansion supplies the adult eligibility pathway, and ESI is not a Medicaid eligibility bar; the engine places the person at 0.00 times FPL." +us,scenario_056,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the $12,000 financial assistance and gross self-employment input in Medicaid MAGI, despite also stating a threshold higher than its own $17,839 total and recognizing the $9,915 HSA deduction. The engine’s Medicaid MAGI level is 0.00 times FPL, so the person is below New Jersey’s ACA adult expansion limit; ESI does not reverse eligibility." +us,scenario_056,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $12,000 financial assistance and the listed self-employment amount as additive Medicaid MAGI and compared the resulting $17,839 shortcut with the expansion limit. PolicyEngine computes the relevant MAGI level as 0.00 times FPL, placing this adult below the New Jersey expansion threshold." +us,scenario_056,head_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model correctly recognized that the stated income total was below its cited expansion threshold, then incorrectly made employer-sponsored insurance disqualifying. ESI is not an exclusion from New Jersey ACA adult Medicaid, so the qualifying adult remains eligible." +us,scenario_056,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model recognized both the ACA expansion adult category and income below 138% FPL but then discarded that pathway because no parent, pregnancy, disability, or elderly category applied. Those other categories are unnecessary: this 20-year-old independently qualifies through New Jersey’s MAGI-based adult expansion, and ESI does not disqualify the person." +us,scenario_056,head_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,"The model imposed a nonexistent rule that access to or enrollment in employer-sponsored insurance eliminates Medicaid eligibility. New Jersey’s ACA adult expansion remains available based on category, immigration status, and MAGI, and the engine places this person at 0.00 times FPL." +us,scenario_056,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model assumed that the facts established no Medicaid pathway and treated ESI as adverse to eligibility. Age 20 and New Jersey residence establish the ACA expansion adult category, while five years since US entry satisfies the modeled immigration rule and 0.00-times-FPL MAGI satisfies the income test." +us,scenario_056,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model looked only for disability, pregnancy, or another traditional categorical pathway and omitted New Jersey’s ACA expansion category for adults under 65. This 20-year-old qualifies through that adult pathway at an engine-computed MAGI level of 0.00 times FPL." +us,scenario_056,head_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"The model treated employer-sponsored insurance and an unspecified income calculation as disqualifying. ESI does not bar Medicaid, and the relevant engine MAGI level is 0.00 times FPL, below New Jersey’s ACA adult expansion threshold." +us,scenario_056,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output. It therefore failed the required output contract rather than completing a Medicaid eligibility calculation. +us,scenario_056,head_medicaid_eligible,minimax-m3,llm_error,health_coverage,False,"The model incorrectly treated ESI as disqualifying and asserted that income exceeded New Jersey’s Medicaid threshold. The engine computes Medicaid MAGI at 0.00 times FPL, so this person satisfies the ACA adult expansion income test, and ESI does not eliminate eligibility." +us,scenario_056,head_medicaid_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model imposed a nonexistent ESI disqualification and incorrectly summed financial assistance, alimony, and the gross self-employment input into $17,839 of Medicaid MAGI. PolicyEngine’s relevant MAGI level is 0.00 times FPL, which is below the New Jersey ACA adult expansion limit." us,scenario_056,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_056,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_056,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model treated non-wage income as a wage-equivalent payroll tax base and applied the 7.65% employee Social Security and Medicare rate. It missed that the only earned-income item listed is self-employment income, which belongs in the separate self_employment_tax output, while alimony and financial assistance are not employee FICA wages." -us,scenario_056,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model stated the correct 92.35% self-employment tax base and 15.3% rate, but its $305 result applies roughly half the required rate. It effectively used an employee-side payroll-tax rate instead of the full self-employment tax rate on net earnings from self-employment." -us,scenario_056,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model treated the self-employment tax calculation as an approximate schedule and submitted a result consistent with applying only a partial Social Security-style rate to the $4,340 income. It missed the full 15.3% self-employment tax on 92.35% of net self-employment income." -us,scenario_056,self_employment_tax,grok-4.3,llm_error,thresholds_rates,False,The model identified the right base and rate structure but rounded the final liability to $612 instead of carrying the engine calculation through to $613.22. Its answer loses the exact 92.35% net-earnings factor and final liability precision required by the requested annual amount. -us,scenario_056,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable self_employment_tax output. It failed the required output contract rather than making a substantive tax-rule calculation. -us,scenario_056,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model misapplied the $400 self-employment threshold by saying $4,340 of SSTB self-employment income produces net earnings below $400. Applying the 92.35% net-earnings factor gives about $4,007.99, so the income is above the threshold and subject to the full 15.3% self-employment tax." -us,scenario_056,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used a $744 maximum excess-shelter deduction instead of the $395.49 deduction applied here, reducing net income to $462 rather than $805.03. That inflated the formula allotment to $159 monthly and bypassed the $95 minimum-allotment result." -us,scenario_056,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model treated $17,839 of annual income as exceeding the SNAP limit without converting it to the $1,486.54 monthly gross-income figure used by the eligibility test. It also treated the mortgage balance and home ownership as disqualifying assets, although the household passes the resource test." -us,scenario_056,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model stopped after the earned-income and standard deductions and omitted the $395.49 excess-shelter deduction. Including it produces $805.03 net income and a $63.18 formula benefit, which is raised to $95 monthly." -us,scenario_056,snap,claude-opus-4.8,llm_error,asset_resource,False,The model inferred disqualifying assets from home ownership and financial assistance even though the listed household passes the SNAP resource test. It also failed to complete the deduction formula that yields $805.03 net income and triggers the $95 monthly minimum. -us,scenario_056,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model compared the $17,839 annual income total directly with SNAP limits and omitted the allowable deductions. Monthly gross income is within the limit, and the standard, earned-income, and excess-shelter deductions reduce net income to $805.03." -us,scenario_056,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the $395.49 excess-shelter deduction and therefore used $1,216 rather than $805.03 as monthly net income. The resulting formula benefit is $63.18 and is raised to the $95 monthly minimum, not reduced to zero." -us,scenario_056,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model incorrectly stated that $1,486.54 of monthly gross income exceeds the applicable gross-income limit; it equals 1.12 times the $1,330 poverty guideline and passes. It also substituted an inference about asset-like resources for the resource test, which the household passes." -us,scenario_056,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model inferred insufficient work hours from annual self-employment income and imposed an immediate ABAWD disqualification. The household passes the applicable work-requirement test, so the income-and-deduction calculation proceeds to a positive benefit." -us,scenario_056,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,The model applied a $672 capped shelter deduction instead of the $395.49 excess-shelter deduction in the trace. This understated net income and overstated the benefit rather than reaching the $63.18 formula amount and $95 minimum. -us,scenario_056,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies it treated $1,486.54 of monthly gross income as above the SNAP threshold. That income is only 1.12 times the $1,330 poverty guideline and passes the gross-income test." -us,scenario_056,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used a maximum excess-shelter deduction that reduced net income to about $489 instead of applying the traced $395.49 shelter deduction. Correct net income is $805.03, making the ordinary allotment $63.18 before the $95 minimum applies." -us,scenario_056,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model effectively reduced countable net income to zero and awarded the maximum allotment. The allowable deductions total $674.49, leaving $805.03 of net income and an expected contribution of $241.50, so only the $95 monthly minimum is payable." -us,scenario_056,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The submitted amount implies net income was understated when applying the standard and shelter deductions. The traced deductions leave $805.03 net income, producing a $63.18 formula allotment that is raised to $95 monthly." -us,scenario_056,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model applied a $733 shelter-deduction cap as the deduction itself, rather than the $395.49 excess-shelter deduction generated by the household calculation. That lowered net income to $471.25 instead of $805.03 and overstated the monthly allotment." -us,scenario_056,snap,gpt-5.4-mini,llm_error,other,False,"The model defaulted to zero despite the prompt specifying the complete household facts and take-up assumption. Those facts establish eligibility and yield $805.03 net income, followed by the $95 monthly minimum allotment." -us,scenario_056,snap,gpt-5.4-nano,llm_error,other,False,The model required unspecified additional low-income indicators instead of applying the stated SNAP inputs. The listed income and deductions establish eligibility and produce the $95 monthly minimum allotment. -us,scenario_056,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly recognized that the formula benefit falls below the minimum but used a $23 federal minimum rather than New Jersey's applicable $95 monthly minimum. This converts the correct minimum-allotment pathway into $276 instead of $1,140 annually." -us,scenario_056,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model left net income high enough to erase the allotment because it did not apply the full $674.49 of allowable deductions. Those deductions produce $805.03 net income and a $63.18 formula amount, which is raised to $95 monthly." -us,scenario_056,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used an excessive capped shelter deduction and arrived at a $159 formula allotment. The traced excess-shelter deduction is $395.49, leaving $805.03 net income and invoking the $95 monthly minimum." -us,scenario_056,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model overstated the excess-shelter deduction, reducing net income enough to calculate $158 monthly. The correct shelter deduction is $395.49, and the resulting $63.18 formula benefit is raised to $95." -us,scenario_056,snap,grok-4.3,llm_error,asset_resource,False,"The model asserted income-and-asset ineligibility even though gross income is 1.12 times the poverty guideline, net income is 0.61 times it, and the resource test is passed. Applying the benefit formula then yields the $95 monthly minimum." -us,scenario_056,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model treated missing utility details as preventing a positive shelter deduction, despite the listed housing expenses supporting a traced $395.49 excess-shelter deduction. The completed calculation yields $805.03 net income and the $95 monthly minimum." -us,scenario_056,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model annualized only the earned-income and standard deductions and omitted the excess-shelter deduction entirely. The full monthly deductions total $674.49, reducing net income to $805.03 and producing a positive allotment subject to the $95 minimum." +us,scenario_056,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented a wage-equivalent payroll-tax base despite the prompt setting unlisted wages to zero. It incorrectly subjected nonwage income or self-employment income to the 7.65% employee Social Security and Medicare rates; with no wages, employee payroll tax is $0, and self-employment income is handled under self_employment_tax." +us,scenario_056,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model stated the correct base adjustment and combined rate but did not perform its own multiplication: $4,340 × 92.35% × 15.3% equals $613.22, not $305. Its answer effectively omitted about half of the combined self-employment tax." +us,scenario_056,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invoked the self-employment tax rate schedule without applying the 15.3% combined rate to the 92.35% net-earnings base. The stated inputs produce $613.22, while $268 does not follow from the required Social Security and Medicare computation." +us,scenario_056,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model used the correct formula but rounded the result incorrectly. $4,340 × 92.35% × 15.3% equals $613.22, not $612." +us,scenario_056,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output or explanation, violating the required output contract." +us,scenario_056,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model reversed the $400 threshold rule: $4,340 of self-employment income is above, not below, $400. Applying the 92.35% net-earnings adjustment and 15.3% combined rate yields $613.22." +us,scenario_056,self_employment_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly declared $4,340 below the $400 self-employment-tax threshold. Because the threshold is exceeded, the 92.35% net-earnings base is taxed at 15.3%, producing $613.22." +us,scenario_056,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model substituted an estimated $744 non-elderly shelter cap for the $395.49 allowable excess-shelter deduction, reducing net income to $462 instead of $805.03. That inflated the formula allotment to $159 monthly and prevented application of the $95 minimum." +us,scenario_056,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model compared $17,839 of annual income as though it exceeded the applicable SNAP limit and treated the mortgage balance as a disqualifying asset. Gross monthly income is $1,486.54, or 1.12 times the poverty guideline, and the household passes both the income and asset tests." +us,scenario_056,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model stopped after the earned-income and standard deductions and omitted the $395.49 excess-shelter deduction. Including all $674.49 of allowable deductions produces $805.03 of net income and a $63.18 formula benefit, which is raised to $95 monthly." +us,scenario_056,snap,claude-opus-4.8,llm_error,asset_resource,False,The model treated homeownership and financial assistance as evidence of disqualifying assets despite the household passing the SNAP asset test. It also failed to complete the deduction calculation that produces $805.03 of net income and triggers the $95 monthly minimum allotment. +us,scenario_056,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model declared $17,839 of annual income above the SNAP limits without converting it to the applicable monthly amount or applying deductions. Gross income is $1,486.54 monthly, and $674.49 of deductions reduce net income to $805.03, within both limits." +us,scenario_056,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model calculated net income using only the earned-income and standard deductions, omitting the $395.49 excess-shelter deduction. Net income is $805.03 rather than $1,216, yielding $63.18 under the ordinary formula and the $95 monthly minimum." +us,scenario_056,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model incorrectly stated that countable income exceeds 130% of poverty and treated financial assistance as an asset-like disqualifier. Gross income is only 1.12 times the poverty guideline, and the household passes the asset test." +us,scenario_056,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model inferred hours worked from annual self-employment income and imposed an immediate ABAWD disqualification. The stated facts do not establish a work-rule failure, and the household passes PolicyEngine's work requirement test." +us,scenario_056,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,The model used an estimated $672 shelter cap and obtained $544.25 of net income instead of applying the $395.49 excess-shelter deduction that yields $805.03. It therefore overstated the formula allotment rather than applying the $95 monthly minimum. +us,scenario_056,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies an annual-versus-monthly threshold comparison or omission of the applicable income tests. Gross income is $1,486.54 monthly, only 1.12 times the poverty guideline, and net income after deductions is $805.03." +us,scenario_056,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used a maximum shelter deduction to reduce net income to about $489 instead of using the $395.49 allowable excess-shelter deduction. Correct net income is $805.03, making the formula benefit $63.18 and invoking the $95 minimum." +us,scenario_056,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model treated the large listed shelter expenses as eliminating all countable net income and awarded the maximum allotment. Allowable deductions total $674.49, leaving $805.03 of net income and an expected contribution of $241.50." +us,scenario_056,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The submitted $140.72 monthly amount implies an overstated shelter deduction and understated net income. The allowable excess-shelter deduction is $395.49, net income is $805.03, and the resulting $63.18 formula benefit is raised to $95." +us,scenario_056,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,The model applied an estimated $733 shelter cap rather than the $395.49 excess-shelter deduction in the computation trace. That reduced net income to $471.25 instead of $805.03 and overstated the formula allotment. +us,scenario_056,snap,gpt-5.4-mini,llm_error,other,False,"The model ignored the instruction that unlisted inputs are zero and that the supplied facts fully define the household, then defaulted to zero instead of calculating eligibility. The provided income and shelter facts produce $805.03 of net income and a $95 monthly minimum benefit." +us,scenario_056,snap,gpt-5.4-nano,llm_error,other,False,"The model demanded additional low-income indicators even though eligibility follows from the listed numeric inputs and the prompt's zero-default rule. Gross income is within the limit, and allowable deductions reduce net income to $805.03." +us,scenario_056,snap,gpt-5.5,llm_error,state_local_rule,False,"The model correctly recognized that the formula benefit falls below the minimum but used a $23 federal minimum instead of New Jersey's $95 monthly minimum allotment for this case. The applicable minimum produces $1,140 annually." +us,scenario_056,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model left countable net income high enough to erase the allotment because it did not apply the full $674.49 of allowable deductions. Those deductions produce $805.03 of net income, a $63.18 formula benefit, and the $95 minimum." +us,scenario_056,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's $159 monthly estimate reflects an excessive capped shelter deduction that reduced net income far below $805.03. The allowable excess-shelter deduction is $395.49, after which the formula amount is below the $95 minimum." +us,scenario_056,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,The model overstated the capped excess-shelter deduction and consequently understated countable net income. Using $395.49 for excess shelter produces $805.03 of net income and the $95 monthly minimum allotment. +us,scenario_056,snap,grok-4.3,llm_error,asset_resource,False,"The model asserted an income-and-asset disqualification even though gross income is within the limit and the household passes the asset test. After $674.49 of deductions, net income is also within the applicable limit." +us,scenario_056,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,The model treated unlisted shelter and utility details as blocking a positive calculation despite the prompt directing all unlisted numeric inputs to zero. The listed inputs generate a $395.49 excess-shelter deduction and $805.03 of net income. +us,scenario_056,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model annualized the earned-income and standard deductions but omitted the excess-shelter deduction entirely. The $395.49 monthly excess-shelter deduction lowers net income to $805.03, so the ordinary formula remains positive and the $95 minimum applies." +us,scenario_056,snap,inkling,llm_error,taxable_income_or_deductions,False,The model used a $712 capped shelter deduction instead of the $395.49 allowable excess-shelter deduction. This understated net income at about $493 rather than $805.03 and overstated the formula benefit to $144 monthly. us,scenario_056,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so the required output was missing." -us,scenario_056,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,The model substituted a $744 maximum shelter cap for the traced $395.49 excess-shelter deduction. This reduced net income to $466.25 instead of $805.03 and inflated the formula allotment above the applicable $95 minimum. -us,scenario_056,snap,minimax-m3,llm_error,thresholds_rates,False,"The model treated the household's income as exceeding the one-person SNAP limit. Monthly gross income of $1,486.54 and net income of $805.03 both pass their respective tests." -us,scenario_056,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model incorrectly compared $17,839 with an annual gross limit of about $19,000 and called it excessive, despite its own figures showing it is lower. It then omitted the $395.49 excess-shelter deduction and the $95 minimum-allotment rule." +us,scenario_056,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model applied a $744 shelter cap as the deduction, while the allowable excess-shelter deduction in this household is $395.49. Its $466.25 net income is therefore too low; the correct $805.03 net income leads to the $95 minimum." +us,scenario_056,snap,minimax-m3,llm_error,thresholds_rates,False,"The model declared income above the one-person limit without applying the correct monthly tests. Gross income is 1.12 times the poverty guideline and net income after deductions is 0.61 times the guideline, so both tests are passed." +us,scenario_056,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model incorrectly stated that $17,839 exceeds an annual gross limit of about $19,000 and then omitted the shelter deduction from net income. Gross income passes, and the full deduction calculation produces $805.03 of net income." +us,scenario_056,snap,qwen3.8-max,llm_error,thresholds_rates,False,The model incorrectly placed gross income above 130% of poverty and calculated net income without the $395.49 excess-shelter deduction. Gross income is 1.12 times the guideline and net income is 0.61 times it. us,scenario_056,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_056,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,The model correctly approximated the $67.74 preliminary New Jersey main tax but reported it as the requested final amount. It failed to apply the New Jersey nonrefundable credits or adjustments that fully offset that liability before refundable credits. -us,scenario_056,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly included the $12,000 financial assistance in New Jersey gross income and deducted both real-estate taxes and charitable donations to manufacture $3,462 of taxable income. The applicable computation uses only $1,498.53 of alimony and $4,340 of self-employment income, subtracts the $1,000 regular exemption, and then applies the credits or adjustments that eliminate the resulting $67.74 preliminary tax." -us,scenario_056,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so it failed the required output contract." -us,scenario_056,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model cycled among incompatible tax bases, including financial assistance as taxable income and subtracting an invented $3,066.18 self-employment-tax deduction and the federal HSA deduction in variants of the New Jersey calculation. The correct New Jersey base is $5,838.53 less the $1,000 regular exemption, and the model also failed to apply the nonrefundable credits or adjustments that reduce the $67.74 preliminary tax to zero." -us,scenario_056,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model incorrectly imported the federal childless-EITC minimum age of 25 into New Jersey eligibility. New Jersey's state-specific childless pathway covers this 20-year-old, producing a $265.60 NJ EITC." -us,scenario_056,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly stated that New Jersey has no state EITC or other refundable income-tax credit. New Jersey's refundable EITC applies here and equals $265.60. -us,scenario_056,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated being age 20 and childless as disqualifying for the NJ EITC. New Jersey permits the childless EITC through its state-specific age rule, so this filer qualifies." -us,scenario_056,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model wrongly concluded that the 20-year-old childless filer could not receive the NJ EITC despite qualifying self-employment earnings. The state-specific childless pathway applies, and the income calculation yields a $265.60 credit." -us,scenario_056,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,The model incorrectly required this 20-year-old to satisfy the federal childless-EITC age rule before receiving the NJ EITC. New Jersey's separate age expansion makes the filer eligible. -us,scenario_056,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model applied the federal age-25 minimum directly to New Jersey's childless EITC. New Jersey's state-specific rule includes this age-20 filer, whose earned income generates $265.60." -us,scenario_056,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,The model incorrectly imposed the federal childless-worker age threshold and inferred possible dependent status even though unlisted status is false. The filer qualifies under New Jersey's age-expanded childless EITC pathway. -us,scenario_056,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model assumed a zero federal EITC necessarily makes the NJ EITC zero. It omitted New Jersey's state-only childless eligibility for younger workers, which produces $265.60 here." -us,scenario_056,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model incorrectly made federal EITC qualification a prerequisite for this younger worker's NJ EITC. New Jersey's childless age expansion covers the 20-year-old filer. -us,scenario_056,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The zero answer omits the New Jersey childless EITC pathway for workers beginning at age 18. Applying that pathway to the listed earned income yields $265.60. -us,scenario_056,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model identified New Jersey's age-18 eligibility rule but incorrectly calculated the credit as 40% of an approximate federal amount. PolicyEngine applies the applicable 8% phase-in and subsequent reduction, yielding $265.60 rather than $123." -us,scenario_056,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly recognized the state-only pathway for ages 18–24 but incorrectly substituted a 40%-of-federal look-alike calculation. The applicable phase-in and reduction produce $265.60, not $123." -us,scenario_056,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model incorrectly treated zero federal EITC and the absence of children as dispositive. New Jersey separately extends its childless EITC to this 20-year-old worker. -us,scenario_056,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,The model carried the federal age-25 childless restriction into the NJ EITC. New Jersey's expanded age rule makes this 20-year-old eligible for $265.60. -us,scenario_056,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The zero answer omits New Jersey's refundable childless EITC for younger workers. The age-20 filer and listed earned income satisfy that pathway and generate $265.60. -us,scenario_056,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model overlooked that the supplied age, state, and earned income are the required low-income eligibility indicators for New Jersey's childless EITC. No qualifying child is required under this pathway." -us,scenario_056,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,The model incorrectly treated the lack of dependents and positive state tax liability as eliminating refundable credits. The NJ EITC is refundable and this childless 20-year-old qualifies independently of pre-credit liability. -us,scenario_056,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model omitted New Jersey's age-expanded childless EITC. Applying it to the filer's earned income produces $265.60. -us,scenario_056,state_refundable_credits,gpt-5.6-sol,llm_error,other,False,The model substituted New Jersey's $50 property-tax credit for the requested refundable-credit total. The operative refundable credit is the $265.60 NJ EITC; the property-tax choice does not replace it. -us,scenario_056,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly found the age-18 childless-worker pathway but incorrectly valued it as 40% of a $332 federal-formula amount. The applicable NJ computation yields $265.60, exactly twice its submitted $132.80." -us,scenario_056,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The zero answer omits New Jersey's childless EITC eligibility for a 20-year-old worker. That pathway produces a refundable credit of $265.60. -us,scenario_056,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model incorrectly assumed that no federal EITC means no NJ EITC. New Jersey's state-specific younger-childless-worker pathway applies despite federal age ineligibility. -us,scenario_056,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model incorrectly limited the NJ EITC to 40% of the actually allowed federal EITC and therefore stopped at zero. It omitted New Jersey's state-only childless eligibility for this 20-year-old; the property-tax deduction choice does not eliminate that credit. -us,scenario_056,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for state_refundable_credits. It therefore failed the required output contract rather than completing the NJ EITC calculation. -us,scenario_056,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,The model incorrectly applied the federal age-25 minimum to New Jersey's childless EITC. New Jersey's younger-worker expansion covers this age-20 filer and generates $265.60. -us,scenario_056,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,The zero answer omits New Jersey's refundable childless EITC pathway for younger workers. The listed age and earned income qualify the filer for $265.60. -us,scenario_056,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly treated the absence of children and the earned-income profile as disqualifying. New Jersey's childless EITC covers the 20-year-old, and $4,033.39 of earned income supports a $265.60 credit." +us,scenario_056,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,The model stopped at the 1.4% preliminary New Jersey tax calculation and reported $67.75. It failed to apply the New Jersey credits or adjustments that fully offset the $67.74 preliminary liability in the requested after-nonrefundable-credits output. +us,scenario_056,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $12,000 of financial assistance in New Jersey gross income and deducted charitable contributions and the full property-tax amount to construct $3,462 of taxable income. The applicable computation uses only $1,498.53 of alimony and $4,340 of self-employment income, subtracts the $1,000 exemption, and then applies the adjustments that eliminate the resulting $67.74 preliminary tax." +us,scenario_056,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for the requested output, so it failed the required output contract." +us,scenario_056,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model improperly treated the $12,000 financial assistance as New Jersey taxable income, invented a $3,066.18 self-employment-tax deduction against state income, and cycled through incompatible HSA and deduction treatments before asserting $299.43 without a supporting calculation. The correct state computation uses $5,838.53 of New Jersey AGI, subtracts the $1,000 exemption, calculates $67.74 of preliminary tax, and fully offsets that liability through applicable credits or adjustments." +us,scenario_056,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model applied the federal minimum age of 25 to New Jersey's childless EITC and missed New Jersey's age-18 eligibility pathway. At age 20, the filer qualifies for the state credit, which yields $265.60." +us,scenario_056,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly stated that New Jersey has no state EITC or other refundable income-tax credit. New Jersey's childless EITC applies at age 20 and produces $265.60 from the filer's earned income. +us,scenario_056,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated being under age 21 as disqualifying for the New Jersey EITC. The state childless-worker pathway begins at age 18, so the 20-year-old qualifies for $265.60." +us,scenario_056,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model wrongly concluded that age 20, childlessness, and unearned income eliminated the New Jersey EITC. The filer satisfies New Jersey's age-18 childless pathway, and adjusted gross income of -$5,882.05 meets the income test, producing $265.60." +us,scenario_056,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model made federal childless EITC age eligibility a prerequisite for the New Jersey credit. New Jersey separately extends its childless EITC to filers age 18 and older, so this 20-year-old receives $265.60." +us,scenario_056,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model applied the federal age-25-to-64 rule directly to New Jersey's childless EITC. New Jersey's minimum age is 18 for this pathway, and the eligible filer's credit is $265.60." +us,scenario_056,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model imposed an age-25 requirement and inferred possible dependent status despite the instruction that unlisted statuses are false. Under New Jersey's age-18 childless-worker rule, this nondependent 20-year-old qualifies for $265.60." +us,scenario_056,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model assumed a zero federal EITC necessarily makes the New Jersey EITC zero. It missed New Jersey's childless EITC extension for ages 18 through 24, which gives this filer $265.60." +us,scenario_056,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model incorrectly required federal EITC qualification and a qualifying child for the New Jersey credit. New Jersey's age-18 childless pathway covers this 20-year-old and yields $265.60. +us,scenario_056,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer omitted New Jersey's age-18 childless EITC pathway. Applying that pathway to $4,033.39 of earned income and the traced reduction produces $265.60." +us,scenario_056,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model correctly identified New Jersey's age-18 childless EITC pathway but incorrectly multiplied the initial federal-formula amount by 40%. PolicyEngine phases in the applicable credit at 8% to $308.55 and then applies the required reduction, yielding $265.60 rather than $123." +us,scenario_056,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,The model correctly recognized the age-18 state pathway but treated the credit as 40% of a federal look-alike amount. The applicable computation phases in at 8% and then reduces the $308.55 preliminary credit to $265.60. +us,scenario_056,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model assumed that zero federal EITC eligibility and no qualifying children eliminate the New Jersey EITC. New Jersey's childless-worker rule starts at age 18, so this filer receives $265.60." +us,scenario_056,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model carried the federal age-25-to-64 restriction into New Jersey's EITC rules. New Jersey admits childless filers beginning at age 18, making this 20-year-old eligible for $265.60." +us,scenario_056,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model failed to identify the New Jersey childless EITC available from age 18. The stated age and earned income establish eligibility and produce a $265.60 refundable credit. +us,scenario_056,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model wrongly required additional low-income indicators or children instead of applying the supplied income and age facts. New Jersey's age-18 childless EITC applies to the 20-year-old and yields $265.60. +us,scenario_056,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the absence of dependents and positive state tax liability as eliminating refundable credits. The New Jersey EITC is refundable without positive liability, and its age-18 childless pathway produces $265.60." +us,scenario_056,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model omitted the qualifying New Jersey childless EITC pathway. At age 20 with $4,033.39 of earned income and qualifying adjusted gross income, the filer receives $265.60." +us,scenario_056,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model substituted New Jersey's $50 property-tax credit for the applicable refundable EITC. The requested output includes the age-18 childless EITC, whose traced computation yields $265.60." +us,scenario_056,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model correctly found the age-18 childless-worker pathway but used a $332 federal-formula base and multiplied it by 40%. The traced calculation uses $4,033.39 of earned income, phases in $308.55 at 8%, and then reduces it to $265.60." +us,scenario_056,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The zero answer omitted New Jersey's refundable childless EITC for filers age 18 and older. Applying it to this 20-year-old yields $265.60. +us,scenario_056,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model incorrectly made a positive federal EITC necessary for the New Jersey EITC. New Jersey's extension covers childless workers ages 18 through 24, producing $265.60 here." +us,scenario_056,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model tied the New Jersey EITC entirely to the zero federal EITC and focused on the property-tax deduction. It missed the state's age-18 childless pathway, which generates a refundable $265.60 credit." +us,scenario_056,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,The model assumed federal EITC ineligibility and no dependents precluded every New Jersey refundable credit. New Jersey's age-18 childless EITC applies and yields $265.60. +us,scenario_056,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required submission contract." +us,scenario_056,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,"The model applied the federal age-25-to-64 rule to New Jersey and therefore set the state EITC to zero. New Jersey's childless EITC begins at age 18, and this filer qualifies for $265.60." +us,scenario_056,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The zero answer omitted New Jersey's age-18 childless EITC. The household composition does not bar the credit, and the filer's earned income produces $265.60." +us,scenario_056,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model wrongly treated the lack of children and the income profile as disqualifying for the New Jersey EITC. The age-18 childless pathway applies to the 20-year-old, and $4,033.39 of earned income leads to $265.60." +us,scenario_056,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly relied on the absence of children, tax liability, and unspecified credit conditions. New Jersey's refundable childless EITC requires neither a child nor positive tax liability, and the supplied age and income produce $265.60." us,scenario_056,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_057,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,The model correctly identified the childless EITC pathway and the plateau but replaced the 2026 maximum with an unsupported $636 lookup-table estimate. Applying the 2026 parameters yields the $664 EITC. -us,scenario_057,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated zero taxable income after the standard deduction as eliminating refundable credits and asserted that the income exceeded the EITC limit. EITC depends on earned income and AGI rather than positive income-tax liability, and this childless joint household receives $664." -us,scenario_057,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model’s reasoning correctly concluded that the head satisfies the joint-return age test and that earnings lie on the childless EITC plateau, but it submitted $397 instead of the maximum its own derivation required. The applicable 2026 plateau amount is $664." -us,scenario_057,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly placed roughly $9,600 of earned income in the married childless EITC phaseout region. That income is below the joint phaseout threshold and produces the 2026 plateau credit of $664, not $218." -us,scenario_057,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model incorrectly reduced the credit to $566 despite recognizing that the joint phaseout begins near $18,000. The household remains on the childless EITC plateau, whose applicable 2026 amount is $664." -us,scenario_057,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model contradicted the 7.65% childless EITC schedule it described by submitting $967, an amount above the applicable maximum. The 2026 plateau credit is $664, and the two spouses are not treated as generating separate worker credits." -us,scenario_057,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,The model used an estimated $649 maximum carried forward from 2025 instead of the applicable 2026 childless EITC parameter. The plateau amount for this calculation is $664. -us,scenario_057,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model projected the 2026 maximum from the 2024 amount and arrived at $630 instead of using the applicable 2026 parameter. The household’s plateau EITC is $664. -us,scenario_057,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly required a qualifying child or different earned-income threshold for EITC eligibility. Childless joint filers qualify when the age and income tests are met, and this household receives $664." -us,scenario_057,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly selected the maximum childless EITC pathway but used $666 as the 2026 maximum. The applicable PolicyEngine parameter produces $664. -us,scenario_057,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model identified the childless EITC but substituted an unsupported $620 estimate for the 2026 benefit. The applicable calculation yields $664. -us,scenario_057,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model selected the correct childless EITC pathway but understated the 2026 amount by $4. The applicable credit is $664. -us,scenario_057,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a fictitious $1,231–$1,283 childless EITC maximum and an internally invalid phase-in calculation. The 7.65% childless schedule is capped at the applicable 2026 maximum of $664." -us,scenario_057,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model ignored that net earnings from self-employment are earned income for EITC purposes and incorrectly treated the absence of wages and children as disqualifying. The 43-year-old head qualifies for the childless EITC, producing $664." -us,scenario_057,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The zero answer omitted the childless EITC pathway triggered by the head’s self-employment earnings and qualifying age. That pathway produces $664 in refundable credits. -us,scenario_057,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model correctly identified eligibility and the plateau but used an estimated maximum of $667. The applicable 2026 maximum is $664. -us,scenario_057,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model correctly placed the household at the childless-worker EITC maximum but used the wrong maximum amount, $649. The applicable 2026 amount is $664." -us,scenario_057,federal_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"The model incorrectly required the 20-year-old spouse to satisfy the childless EITC age test. On a joint return, eligibility is established because the 43-year-old head meets the age requirement, resulting in a $664 credit." -us,scenario_057,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,The model incorrectly concluded that income phaseouts reduce the childless EITC to zero. The household’s income remains within the eligible joint-filer range and yields $664. -us,scenario_057,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly applied the age test, net self-employment earnings, and plateau logic but used a projected maximum of $667. The applicable 2026 plateau amount is $664." -us,scenario_057,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,The model correctly identified the maximum-credit region but used $648 as the 2026 childless EITC maximum. The applicable maximum is $664. -us,scenario_057,federal_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"The model incorrectly required both spouses on a joint return to be at least 25 for the childless EITC. The 43-year-old head satisfies the age test, so the household receives $664." -us,scenario_057,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model incorrectly treated $10,400 of self-employment income as above the childless joint-filer EITC limit. It falls within the qualifying range and produces $664." -us,scenario_057,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly treated low income and disability benefits as eliminating EITC eligibility. The head’s self-employment income is qualifying earned income, the disability amounts do not disqualify the household, and the resulting childless EITC is $664." -us,scenario_057,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model looked only at the disabled flag and disability-benefit amount and failed to use the SSI receipt implied by the household's SSI benefit. Under Louisiana rules, SSI receipt places the head in the SSI_RECIPIENT Medicaid category, which confers automatic Medicaid eligibility independent of the disabled flag or income screen." -us,scenario_057,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model enumerated MAGI and demographic pathways such as expansion, pregnancy, parenthood, and age 65+, but omitted the SSI-recipient Medicaid category. The head receives SSI, and that categorical status automatically establishes Louisiana Medicaid eligibility even for a 43-year-old adult with no dependent children." -us,scenario_057,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated disability alone as sufficient for Medicare regardless of age. It omitted the requirement for an under-65 qualifying pathway such as the requisite period of SSDI entitlement, which is not established by the household facts." -us,scenario_057,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model converted generic disability-benefit receipt into Medicare eligibility. Those benefits do not establish SSDI entitlement or completion of Medicare's 24-month disability waiting period, so the 43-year-old head does not qualify through that pathway." -us,scenario_057,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly named the SSDI waiting-period rule but then ignored it, inferring qualifying Social Security disability entitlement from a $14 generic disability-benefits input. The facts establish neither SSDI entitlement nor 24 months of receipt, so the under-65 disability pathway is not met." -us,scenario_057,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model inferred SSDI or SSI receipt from the generic disability-benefits field and treated either as proof of Medicare eligibility. The facts establish neither SSDI entitlement nor completion of the 24-month waiting period, and SSI receipt alone does not supply that Medicare pathway." -us,scenario_057,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated blindness, disability, and generic disability benefits as a modeled under-65 Medicare pathway. None establishes the required SSDI entitlement and waiting period, leaving the age-65 test unmet at age 43." -us,scenario_057,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model equated listed disability-benefit receipt with the disability-beneficiary rule for Medicare. It failed to require established SSDI entitlement for the requisite duration, which the prompt does not provide." -us,scenario_057,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model assumed PolicyEngine makes the disability boolean itself an under-65 Medicare eligibility pathway. Disability alone is insufficient without a specified qualifying basis such as the requisite period of SSDI entitlement, and the head is only 43." +us,scenario_057,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,The model correctly identified the childless EITC plateau but replaced the 2026 result with an unsupported lookup-table estimate of $636. Applying the 2026 parameters to this joint return yields the $664 EITC. +us,scenario_057,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated zero taxable income after the standard deduction as preventing refundable credits. EITC depends on earned income and AGI rather than positive income-tax liability, and the head's self-employment earnings generate a $664 childless EITC." +us,scenario_057,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model's reasoning placed the household on the childless-EITC plateau and computed about $649, but it then submitted an unexplained $397. The applicable 2026 plateau amount is $664, with no computation in its reasoning supporting the submitted value." +us,scenario_057,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model incorrectly placed roughly $9,665 of joint-return income in the childless-EITC phaseout region. The married-filing-jointly phaseout begins above this income, so the 2026 calculation produces $664 rather than $218." +us,scenario_057,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly reduced the credit to $566 even though it acknowledged that the joint-filer phaseout begins near $18,000. This household's income remains below that phaseout threshold, and the 2026 childless EITC is $664." +us,scenario_057,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model contradicted its own 7.65% childless-EITC schedule by submitting $967, an amount exceeding the applicable maximum. The 2026 joint childless calculation is capped at $664." +us,scenario_057,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model substituted an estimated $649 maximum derived from 2025 parameters for the applicable 2026 amount. The household is on the relevant plateau, where the 2026 EITC equals $664." +us,scenario_057,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model informally inflated the 2024 maximum to an estimated $630 instead of applying the 2026 EITC parameters. Those parameters yield a $664 childless EITC. +us,scenario_057,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly denied EITC eligibility despite the head's qualifying age and earned self-employment income. A qualifying child is not required for the childless EITC, which equals $664 here." +us,scenario_057,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly identified maximum childless-EITC eligibility but used $666 instead of the applicable 2026 amount. The correct parameterized result is $664. +us,scenario_057,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used an unsupported $620 estimate for the 2026 childless EITC. Applying the actual 2026 schedule to this eligible joint household yields $664. +us,scenario_057,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model correctly recognized the childless EITC but used $660 rather than the exact 2026 parameterized result. The refundable credit is $664. +us,scenario_057,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model invented a $1,231-to-$1,283 childless-EITC maximum that is incompatible with the 7.65% schedule and also changed earned income to $8,320 without support. The applicable 2026 childless EITC is $664." +us,scenario_057,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model ignored that net earnings from self-employment are earned income for EITC purposes. No qualifying child is required for the childless EITC, and the age-43 head qualifies the joint return for $664." +us,scenario_057,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The zero answer omits the childless EITC generated by the head's qualifying self-employment earnings and age. The household receives $664 even though it has no qualifying children. +us,scenario_057,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model correctly placed the household at the maximum childless EITC but used an estimated maximum of $667. The applicable 2026 amount is $664. +us,scenario_057,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,The model correctly found maximum childless-worker EITC eligibility but substituted a $649 maximum. The 2026 joint-filer calculation yields $664. +us,scenario_057,federal_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"The model incorrectly required the 20-year-old spouse to satisfy the childless-EITC age test. On a joint return, the age-43 head satisfies the age condition, so the household receives $664." +us,scenario_057,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly asserted that income phaseouts eliminate the EITC. The household's income is below the married-filing-jointly childless phaseout range, producing a $664 credit." +us,scenario_057,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly identified the qualifying age, net self-employment earnings, and plateau treatment but used a projected $667 maximum. The exact 2026 EITC is $664." +us,scenario_057,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,The model correctly concluded that income is below the joint phaseout threshold but used $648 as the 2026 maximum. The applicable plateau amount is $664. +us,scenario_057,federal_refundable_credits,inkling,llm_error,age_disability,False,"The model applied the age-25 minimum to the spouse as though both joint filers had to pass it. The head is 43 and satisfies the joint return's childless-EITC age condition, yielding $664." +us,scenario_057,federal_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"The model incorrectly required both spouses on a joint return to be at least 25. The age-43 head satisfies the childless-EITC age rule, so the eligible household receives $664." +us,scenario_057,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model incorrectly treated $10,400 of self-employment income as above the childless-EITC limit. For married filing jointly, this income is below the phaseout threshold and produces a $664 EITC." +us,scenario_057,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly claimed that low income and disability benefits defeat EITC eligibility. The self-employment earnings count as earned income, the small disability-benefit amounts do not eliminate the credit, and the result is $664." +us,scenario_057,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the absence of wages and qualifying children as eliminating EITC. Self-employment earnings count as earned income, and the age-43 head qualifies for the childless EITC of $664." +us,scenario_057,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model looked only to the disability flag and disability-benefit amount and failed to account for the head's SSI receipt. SSI receipt places the head in Louisiana's automatically eligible SSI_RECIPIENT Medicaid category. +us,scenario_057,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model restricted Medicaid eligibility to expansion, pregnancy, parent/caretaker, and aged pathways and omitted automatic SSI-recipient eligibility. The head's $7,231.00146484375 in SSI establishes eligibility through the SSI_RECIPIENT category regardless of those demographic categories or the MAGI test." +us,scenario_057,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated disability alone as sufficient for Medicare regardless of age. A disability flag does not establish the required SSDI entitlement and 24-month waiting period, and the 43-year-old head does not meet the age-65 rule." +us,scenario_057,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model converted generic disability-benefit receipt into automatic under-65 Medicare eligibility. The listed $14 does not establish SSDI entitlement or 24 months of entitlement, while the head is only 43." +us,scenario_057,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly named the SSDI pathway but omitted its required 24-month entitlement period and treated the generic $14 disability-benefit amount as proof of qualifying Social Security disability benefits. Neither SSDI entitlement nor its duration is listed, so those inputs are false or zero under the prompt." +us,scenario_057,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model inferred both SSDI or SSI receipt and completion of the 24-month SSDI waiting period from a generic disability-benefit amount. SSI does not itself confer Medicare, and no SSDI entitlement or 24-month duration was provided." +us,scenario_057,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated blindness, disability, and generic disability benefits as a complete modeled Medicare pathway. These facts do not supply qualifying SSDI entitlement or the required 24 months, and the head is below age 65." +us,scenario_057,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model equated unspecified disability-benefit receipt with satisfying the disability-beneficiary Medicare rule. The rule requires qualifying SSDI entitlement and completion of the waiting period, neither of which appears in the household facts." +us,scenario_057,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model assumed PolicyEngine’s under-65 Medicare pathway is triggered directly by the disability boolean. Disability alone does not establish qualifying SSDI entitlement or 24 months of entitlement, so the 43-year-old head remains ineligible." us,scenario_057,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_057,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model used the right shortcut, 92.35% of $10,400 times 15.3%, but stopped at a rounded back-of-envelope result of $1,464 instead of carrying the engine's exact self-employment tax computation and rounding to $1,469.47. Its answer is a numerical approximation error, not a different eligibility or income classification." -us,scenario_057,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model incorrectly excluded the listed SSTB self-employment income from self-employment tax. SSTB status affects qualified business income deduction treatment, not whether net self-employment earnings are subject to SECA Social Security and Medicare tax." -us,scenario_057,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable self_employment_tax value or explanation. The required output was an annual numeric self-employment tax amount, and the correct derivation taxes the listed self-employment income under SECA rather than omitting the field." -us,scenario_057,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model incorrectly added 0.9% Additional Medicare Tax to self-employment earnings above $400. The $400 rule is the filing/liability threshold for self-employment tax, while Additional Medicare Tax applies only above the high-income statutory threshold for the filing unit, which this joint household is far below." -us,scenario_057,self_employment_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model invented a $12,000 threshold for self-employment tax liability and zeroed out the tax because $10,400 was below it. Self-employment tax applies once net earnings from self-employment exceed the small statutory filing threshold, so this household's listed self-employment income is taxable under SECA." -us,scenario_057,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the approximately $602.58 monthly SSI benefit generated for the disabled household members and counted only the $38 of explicitly listed disability benefits. That omission reduced its monthly net income from $1,090.08 to roughly $485 and consequently overstated SNAP." -us,scenario_057,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model acknowledged that the household had low income and assets but then assigned zero without applying the eligibility tests or allotment formula. The household passes the net and gross income tests, has only $911.76 in assets, and is categorically eligible through SSI receipt." -us,scenario_057,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the approximately $602.58 per month of calculated SSI from SNAP countable income and then submitted $6,240 despite its own intermediate calculation producing about $3,990. Correct monthly net income is $1,090.08, so the household contribution is about $327 rather than the much smaller contribution implicit in its answer." -us,scenario_057,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted calculated SSI from countable income, inconsistently converted the listed $38 annual disability benefits into $456 of annual unearned income, and then submitted an amount unsupported by its own $4,536 calculation. SNAP countable income includes approximately $602.58 of monthly SSI, yielding $1,090.08 of monthly net income." -us,scenario_057,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model used an erroneous two-person maximum of about $304 per month and treated net income as nearly zero. The applicable monthly maximum is $546 for most months and $558.24 later, while $1,090.08 of net income generates a substantial expected contribution." -us,scenario_057,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the approximately $602.58 monthly SSI benefit from SNAP countable income and instead treated the household as receiving only $38 annually in unearned disability income. It also introduced an unsupported 40% self-employment business-expense deduction, driving net income down to $205.17 per month instead of $1,090.08." -us,scenario_057,snap,claude-sonnet-5,llm_error,period_annualization,False,"The model mixed annual and monthly quantities, treating the $10,400 annual self-employment amount as though its resulting countable income were roughly $2,000 per month. It also omitted the calculated SSI amount and therefore never formed the traced monthly gross income of $1,472.42 or net income of $1,090.08." -us,scenario_057,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model counted only self-employment and the listed disability benefits, omitting approximately $602.58 per month of calculated SSI. This produced net income of $489.34 rather than $1,090.08 and understated the expected household contribution." -us,scenario_057,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included SSI but used an overstated annual SSI amount of $13,122.96 instead of the traced benefit of approximately $602.58 per month. It also applied guessed annual deductions and an unsupported medical deduction, producing excessive net income and too little SNAP." -us,scenario_057,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer incorrectly fails SNAP eligibility. The household passes the gross and net income tests, passes the asset test, and is categorically eligible through SSI, after which the allotment formula produces a positive benefit." -us,scenario_057,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model’s $76 monthly result implies that it overstated countable income or understated deductions after including SSI. The traced monthly net income is $1,090.08, which produces $219 in most months and $232.74 in later months." -us,scenario_057,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used $12,729 of annual unearned income, substantially exceeding the calculated SSI plus listed disability benefits. This inflated the household contribution and reduced SNAP to $776 instead of applying the allotment formula to $1,090.08 of monthly net income." -us,scenario_057,snap,gemini-3.6-flash,llm_error,other,False,"The model recognized eligibility but assigned a flat $268 monthly benefit without deriving the household contribution from net income or accounting for the within-year parameter change. The correct monthly results are generally $219 and later $232.74, totaling $2,669.22." -us,scenario_057,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model treated unlisted prior SNAP receipt as a command to return zero, despite the prompt directing it to assume program take-up and estimate the benefit. Applying the eligibility tests and allotment calculation yields positive SNAP." -us,scenario_057,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly declared counted income too high. Monthly gross income of $1,472.42 and net income of $1,090.08 equal only 84% and 62% of the respective poverty guideline, and SSI receipt independently establishes categorical eligibility." -us,scenario_057,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model confused an unlisted input benefit amount with the requested calculated output and ignored the instruction to assume take-up. SNAP eligibility and the allotment must be computed from the household facts, producing a positive annual benefit." -us,scenario_057,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model omitted approximately $602.58 per month of calculated SSI from SNAP countable income. Its annual net income of $5,862 is therefore far below the traced monthly net income of $1,090.08, causing it to understate the expected contribution and overstate SNAP." -us,scenario_057,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model’s $340 monthly estimate understates countable net income and the resulting 30% household contribution. With calculated SSI included, monthly net income is $1,090.08 and the benefit is generally $219, increasing to $232.74 later in the year." -us,scenario_057,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model’s $70 monthly result implies excessive countable income after deductions. The correct net-income budget is $1,090.08 per month, producing a $327 contribution against the $546 maximum and therefore $219 in most months." -us,scenario_057,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model included SSI but overstated countable monthly income after the earned-income and standard deductions, reducing the allotment to $70. The traced net income is $1,090.08, which yields $219 in most months and $232.74 later." -us,scenario_057,snap,grok-4.3,llm_error,categorical_eligibility,False,"Neither income nor assets exceed the applicable SNAP limits: gross income is 84% of the guideline, net income is 62%, and assets are $911.76. SSI receipt also establishes categorical eligibility, so the benefit does not calculate to zero." -us,scenario_057,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model overstated monthly SSI and total gross income at about $1,928, leading to net income near $1,551. The traced gross and net amounts are $1,472.42 and $1,090.08, respectively, which produce a materially larger allotment." -us,scenario_057,snap,grok-build-0.1,llm_error,period_annualization,False,"The model subtracted a $209 monthly standard deduction only once from annual income and omitted calculated SSI entirely. SNAP is budgeted monthly, with $1,472.42 of gross income and $1,090.08 of net income determining each month’s contribution." -us,scenario_057,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so it failed the required output contract before any substantive SNAP calculation could be evaluated." -us,scenario_057,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used $13,183 of estimated annual SSI, substantially above the traced SSI amount of approximately $602.58 per month. That inflated monthly net income to $1,586.08 rather than $1,090.08 and reduced the allotment excessively." -us,scenario_057,snap,minimax-m3,llm_error,asset_resource,False,"The model reversed the asset comparison: $912 is below, not above, its own cited asset limit of about $4,500. The traced asset amount is $911.76, so the household passes the asset test and receives positive SNAP." -us,scenario_057,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly treated SSI eligibility as excluding all income of SSI recipients from SNAP countable income. SSI receipt confers categorical eligibility, but the SSI payment and countable self-employment income still enter the SNAP budget; it also overstated the two-person maximum at $635 per month." -us,scenario_057,spouse_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated Louisiana CHIP as an income-tested program covering low-income adults. The 20-year-old spouse is already Medicaid-eligible in the adult category, which automatically excludes the spouse from CHIP regardless of income." -us,scenario_057,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply Louisiana's MAGI adult expansion category and treated the absence of extra qualifying details as disqualifying. The spouse's age 20 status and 0.45 FPL household MAGI are sufficient for the adult expansion pathway, so the submitted 0 omits the controlling eligibility route." -us,scenario_057,spouse_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly required pregnancy or parenthood of a minor child for Louisiana Medicaid eligibility and excluded the ACA adult expansion category. Under the adult expansion pathway, the spouse is a non-dependent adult under 65 with MAGI at 0.45 FPL, so pregnancy and parent/caretaker status are irrelevant to eligibility." -us,scenario_057,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated the generic disability flag as automatic Medicare eligibility regardless of age. The spouse is 20, and the listed facts do not establish an under-65 Medicare entitlement, so the age-based computation returns no eligibility." -us,scenario_057,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model inferred SSDI entitlement and completion of Medicare's 24-month disability waiting period from a generic disability flag and $24 of unspecified disability benefits. Those facts were not listed, and the prompt expressly required unlisted statuses to be false, so the 20-year-old spouse does not qualify." -us,scenario_057,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model incorrectly converted reported disability and unspecified disability-benefit receipt into Medicare eligibility under a disability pathway. Neither fact establishes the required Medicare disability entitlement, while the spouse is below age 65." -us,scenario_057,spouse_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model treated any disability-benefit receipt as satisfying Medicare's disability-beneficiary rule. The $24 amount is not identified as qualifying Social Security disability entitlement, and the 20-year-old spouse fails the applicable age test." -us,scenario_057,spouse_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model invented a PolicyEngine rule under which the generic disabled status alone satisfies an under-65 Medicare pathway. PolicyEngine's computation does not activate that pathway from this input, and age 20 is below the eligibility threshold." -us,scenario_057,spouse_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented categorical WIC pathways that do not apply: WIC does not cover nonpregnant 20-year-old disabled adults, and disability is not a substitute for pregnancy, postpartum, breastfeeding, infancy, or being a child under age 5. It also ignored the prompt rule that unlisted statuses are false, so the spouse is not pregnant, postpartum, or breastfeeding and fails WIC before income testing." -us,scenario_057,ssi,claude-fable-5,llm_error,categorical_eligibility,False,"The model submitted a placeholder zero without applying the head's blindness/disability eligibility, resource test, or income calculation. The head qualifies and receives $7,231 after countable income reduces the individual benefit." -us,scenario_057,ssi,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model incorrectly treated both spouses as SSI-eligible and evaluated a pooled couple benefit. SSI is individual here: the spouse fails the aged, blind, or disabled criterion, while the eligible head receives $7,231 after spousal deeming and the applicable income exclusions." -us,scenario_057,ssi,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model awarded a couple-level SSI benefit after declaring both spouses eligible. The spouse fails the SSI aged, blind, or disabled criterion, so the calculation must use the head's individual benefit and countable income with spousal deeming, yielding $7,231." -us,scenario_057,ssi,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model used the eligible-couple federal benefit rate and treated the spouse's stated disability as sufficient SSI eligibility. PolicyEngine finds only the head SSI-eligible and computes the head's individual payment with deemed spousal income, producing $7,231 rather than a couple award." -us,scenario_057,ssi,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated roughly $9,600 of self-employment earnings as countable and therefore eliminated the benefit. SSI earned-income exclusions and disregards, together with spousal deeming, leave the head with $391.42 in monthly countable income and a $7,231 annual payment." -us,scenario_057,ssi,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model applied an estimated couple federal benefit rate because it treated both adults as SSI-eligible. The spouse fails the SSI aged, blind, or disabled test, and the head must receive an individual award calculated with spousal deeming, yielding $7,231." -us,scenario_057,ssi,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The reasoning itself derived a positive couple payment but the model submitted zero, and the derivation also wrongly treated both spouses as eligible for a couple award. Only the head qualifies; the individual calculation with spousal deeming produces $7,231." -us,scenario_057,ssi,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model subtracted countable income from an estimated eligible-couple federal benefit rate. The spouse is not SSI aged, blind, or disabled under the applicable eligibility variable, so only the head's individual award applies and equals $7,231 after deeming." -us,scenario_057,ssi,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model assumed both spouses qualified and began with the married-couple maximum. The spouse fails SSI's aged, blind, or disabled criterion; the head's individual benefit after spousal deeming and $391.42 of monthly countable income is $7,231." -us,scenario_057,ssi,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the head's earnings as disqualifying without applying SSI's countable-income computation. The resulting monthly countable income is $391.42, below the individual benefit level, leaving $7,231 payable annually." -us,scenario_057,ssi,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model explicitly used the SSI couples maximum and pooled the spouses' status and income. Only the head is SSI-eligible, and the individual calculation with spousal deeming yields $7,231." -us,scenario_057,ssi,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model declared both adults eligible and reduced a couple federal benefit rate by pooled countable income. The spouse fails the SSI aged, blind, or disabled test, so the correct unit is the eligible head with spousal deeming and an annual award of $7,231." -us,scenario_057,ssi,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model treated the two adults as an eligible SSI couple solely because both were described as disabled. PolicyEngine's SSI eligibility criterion is satisfied only by the head, whose individual benefit after spousal deeming is $7,231." -us,scenario_057,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model ignored the prompt's take-up assumption and treated the absence of listed SSI receipt as grounds for zero. It also treated $10,400 of self-employment income as disqualifying instead of calculating $391.42 in monthly countable income, which leaves the eligible head $7,231." -us,scenario_057,ssi,gpt-5.4-mini,llm_error,asset_resource,False,"The model falsely stated that $912 of bank assets exceeds the SSI asset limit. The head passes the resource test with $911.76 in countable resources and, after the income calculation, receives $7,231." -us,scenario_057,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required SSI receipt or eligibility to be explicitly listed despite the instruction to calculate eligibility and assume take-up. The head's blindness/disability, qualifying resources, and countable-income result establish a $7,231 annual SSI payment." -us,scenario_057,ssi,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model applied the eligible-couple rate and treated the spouse as SSI-eligible. The spouse fails the aged, blind, or disabled criterion, while the head receives an individual benefit with spousal deeming of $7,231." -us,scenario_057,ssi,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model treated both adults as eligible and used a couple federal benefit rate. SSI is calculated for the eligible head alone with spousal deeming because the spouse fails the aged, blind, or disabled criterion; the result is $7,231." -us,scenario_057,ssi,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model incorrectly asserted that both spouses met disability-based SSI eligibility and reduced a couple maximum. Only the head meets the SSI aged, blind, or disabled criterion, and the head's individual deemed-income calculation yields $7,231." -us,scenario_057,ssi,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model used the SSI couple limit and couple benefit because it treated both adults as eligible. The spouse fails the controlling aged, blind, or disabled test; only the head receives SSI, totaling $7,231 after spousal deeming." -us,scenario_057,ssi,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated self-employment income as exceeding the benefit limit without applying the full SSI countable-income rules. Those rules leave $391.42 in monthly countable income for the eligible head and a $7,231 annual payment." -us,scenario_057,ssi,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model applied a projected couple federal benefit rate after classifying both spouses as eligible. The spouse fails the SSI aged, blind, or disabled criterion, so the head's individual payment with spousal deeming is $7,231." -us,scenario_057,ssi,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model calculated SSI from an eligible-couple annual maximum and pooled exclusions across both spouses. Only the head is eligible, and the required individual calculation with spousal deeming produces $7,231." -us,scenario_057,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SSI value or explanation, violating the required output contract." -us,scenario_057,ssi,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model treated both spouses as eligible and used the couple benefit rate and resource limit. The spouse fails SSI's aged, blind, or disabled criterion, while the eligible head's individual benefit after deeming is $7,231." -us,scenario_057,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model inferred nonreceipt from the small listed disability-benefit amounts instead of calculating SSI under the take-up assumption. The head independently satisfies the blindness/disability and resource tests and receives $7,231 after the income reduction." -us,scenario_057,ssi,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model alternated among individual and couple calculations, treated the spouse as SSI-eligible, and used invented federal benefit rates. The spouse receives zero because the aged, blind, or disabled criterion is false; the head's individual calculation with spousal deeming yields $7,231." -us,scenario_057,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model computed Louisiana tax from federal AGI minus only two $4,500 personal exemptions, leaving $665 taxable income, and ignored the full Louisiana deduction structure that eliminates taxable income for this low-income married filing jointly household. Its $12 answer is exactly the result of taxing that invented $665 base at 1.75%, so the error is the state taxable-income calculation, not the state aggregation step." -us,scenario_057,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable value or explanation for state_income_tax_before_refundable_credits. This is a missing-output contract failure rather than a substantive Louisiana tax calculation. -us,scenario_057,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model recognized Louisiana's 5% EITC and the childless federal EITC pathway but discarded both without calculating them. Self-employment is earned income, the household receives a $664 federal EITC, and the refundable Louisiana credit is $33.20." -us,scenario_057,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model overlooked Louisiana's refundable EITC and incorrectly treated dependents or special expenses as necessary. A childless married couple can receive the federal EITC, and Louisiana refunds 5% of the $664 federal credit." -us,scenario_057,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly required a qualifying child or child-related expense for Louisiana's EITC. The childless EITC pathway applies to the household's self-employment earnings, producing a $664 federal EITC and a $33.20 Louisiana credit." -us,scenario_057,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model recognized the 5% Louisiana EITC but incorrectly rounded a nonzero childless credit to zero. The federal EITC is $664 after applying the childless married-filer schedule, so 5% equals $33.20." -us,scenario_057,state_refundable_credits,claude-opus-5,llm_error,other,False,"The model misclassified Louisiana's 5% EITC as nonrefundable or otherwise limited to zero. It is refundable, and 5% of the household's $664 federal EITC equals $33.20." -us,scenario_057,state_refundable_credits,claude-sonnet-4.6,llm_error,other,False,"The model incorrectly stated that Louisiana has no state EITC or applicable refundable credit. Louisiana refunds an earned income credit equal to 5% of the federal EITC, yielding $33.20 from the household's $664 federal credit." -us,scenario_057,state_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model incorrectly stated that Louisiana lacks an applicable refundable EITC and treated the absence of dependents as disqualifying. Louisiana's refundable credit equals 5% of the $664 childless federal EITC, or $33.20." -us,scenario_057,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used the correct 5% Louisiana rate but understated the federal EITC as $649. The federal EITC calculation yields $664, making the state credit $33.20 rather than $32.45." -us,scenario_057,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model used the correct 5% Louisiana rate but understated the federal EITC as $630. Applying the federal EITC schedule produces $664, so the Louisiana credit is $33.20." -us,scenario_057,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The zero answer omits Louisiana's refundable EITC. The household receives a $664 federal EITC from its self-employment earnings, and Louisiana refunds 5% of it, or $33.20." -us,scenario_057,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly denied eligibility for all Louisiana refundable credits. The household qualifies for the childless federal EITC, and Louisiana's refundable 5% match produces $33.20." -us,scenario_057,state_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model treated the spouse's age of 20 as disqualifying the couple from the childless EITC. The head is age 43 and satisfies the applicable age condition, so the $664 federal EITC generates a $33.20 Louisiana credit." -us,scenario_057,state_refundable_credits,gpt-5.4-mini,llm_error,other,False,The model omitted Louisiana's refundable 5% EITC match. The household's $664 federal EITC triggers a $33.20 state refundable credit. -us,scenario_057,state_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model treated the absence of an explicitly listed credit as an input defaulting to zero instead of applying Louisiana tax law. Louisiana refunds 5% of the calculated $664 federal EITC, yielding $33.20." -us,scenario_057,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly required children for an applicable Louisiana refundable credit. The federal EITC has a childless pathway, and Louisiana refunds 5% of the resulting $664 credit, or $33.20." -us,scenario_057,state_refundable_credits,gpt-5.6-terra,llm_error,other,False,The model failed to derive the Louisiana EITC from the federal EITC rather than from an explicit household eligibility flag. The $664 federal EITC produces a refundable state credit of $33.20. -us,scenario_057,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no Louisiana refundable credit applies. The childless federal EITC applies to the household's self-employment earnings, and Louisiana's 5% match equals $33.20." -us,scenario_057,state_refundable_credits,grok-4.5,llm_error,other,False,"The model incorrectly stated that Louisiana provides no applicable refundable income-tax credit. Its refundable EITC equals 5% of the household's $664 federal EITC, producing $33.20." -us,scenario_057,state_refundable_credits,grok-build-0.1,llm_error,other,False,"The model omitted Louisiana's refundable EITC from the credits applicable to this household. The state credit is 5% of the $664 federal EITC, or $33.20." +us,scenario_057,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model stated the correct 92.35% base and 15.3% rate but miscomputed their product. Applying them to $10,400 yields $1,469.47, not $1,464.00." +us,scenario_057,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model incorrectly excluded SSTB self-employment income from the self-employment-tax base. The listed $10,400 is self-employment income subject to the 92.35% net-earnings factor and the 15.3% combined Social Security and Medicare rate." +us,scenario_057,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self-employment-tax output. The required computation produces $1,469.47 from $10,400 × 92.35% × 15.3%." +us,scenario_057,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model correctly computed the ordinary self-employment tax and then wrongly imposed the 0.9% Additional Medicare Tax above $400. The $400 figure is the net-earnings threshold for self-employment-tax filing, while the Additional Medicare threshold is $250,000 for joint filers and that tax is expressly excluded from this output." +us,scenario_057,self_employment_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model invented a $12,000 exemption threshold for self-employment tax. The applicable net-earnings threshold is $400, so $10,400 of self-employment income generates $1,469.47 of tax after the 92.35% adjustment." +us,scenario_057,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model correctly calculated $1,469.47 and then reported approximately one-half of it. One-half of self-employment tax is deductible in computing adjusted gross income; it does not reduce the self-employment-tax liability reported by this output." +us,scenario_057,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,The model omitted the household’s calculated SSI of $602.58 per month from SNAP income and treated only the listed $38 of disability benefits as unearned income. That understated monthly net income and inflated the allotment from the correct mostly-$219 monthly amount to roughly $380. +us,scenario_057,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced self-employment income by 92.35% instead of applying the SNAP income calculation, then asserted a zero benefit despite describing the household as low-income and eligible. The resulting gross and net income remain below the applicable limits and generate a positive allotment." +us,scenario_057,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the $602.58 monthly SSI amount from countable SNAP income and then submitted $6,240 despite its own calculation producing about $3,990. With SSI included, monthly net income is $1,090.08 and the household contribution is about $327, not near zero." +us,scenario_057,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted the calculated SSI benefit from SNAP income, inconsistently converted the $38 annual disability-benefit input to $456 of annual unearned income, and then submitted an amount unrelated to its own $4,536 calculation. These errors substantially understated the expected household contribution." +us,scenario_057,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model used an erroneous two-person maximum allotment of about $304 per month and treated net income as nearly zero. The applicable maxima are $546 and later $558.24, while $1,090.08 of monthly net income produces a substantial 30% contribution." +us,scenario_057,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the $602.58 monthly SSI amount from SNAP income and invented a Louisiana 40% self-employment business-expense deduction for this calculation. Those choices reduced its monthly net income to $205.17 instead of $1,090.08 and inflated the benefit." +us,scenario_057,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model applied unrelated self-employment-tax factors and a 50% deduction to SNAP self-employment income, while also describing annual inputs as monthly amounts. It therefore failed to derive the traced $1,472.42 gross and $1,090.08 net monthly income used for the allotment." +us,scenario_057,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model counted only self-employment income and the listed $38 of disability benefits, omitting $602.58 of monthly SSI. This reduced monthly net income from $1,090.08 to $489.34 and understated the household’s 30% contribution." +us,scenario_057,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included $13,122.96 of annual SSI, far above the $602.58 monthly amount generated for this household, and also invented a $60 medical deduction from only $40 of annual expenses. The overstated SSI income drove its SNAP allotment far below the traced amount." +us,scenario_057,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,The zero answer treats the household as failing SNAP eligibility even though gross income is 84% and net income is 62% of the applicable poverty guideline. Passing those tests and categorical eligibility produces a positive monthly allotment. +us,scenario_057,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model’s $76 monthly result reflects excessive countable income after including SSI. The generated SSI is $602.58 per month, and the complete deduction calculation yields $1,090.08 of net monthly income and benefits of mostly $219 per month." +us,scenario_057,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model counted $12,729 of unearned SSI income rather than the household’s generated $602.58 monthly SSI amount. That overstated net income and reduced the allotment to $776 instead of applying a roughly $327 monthly household contribution." +us,scenario_057,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model asserted a $268 monthly benefit without performing the maximum-allotment-minus-30%-of-net-income calculation. Net monthly income of $1,090.08 yields mostly $219 per month, with later months at $232.74." +us,scenario_057,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model treated an unlisted current SNAP receipt as zero instead of calculating eligibility and assuming take-up as explicitly instructed. The household passes the income, asset, categorical-eligibility, and work tests and therefore receives a positive calculated benefit." +us,scenario_057,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model incorrectly classified the household’s income as too high. Monthly gross income of $1,472.42 is 84% of the poverty guideline and monthly net income of $1,090.08 is 62%, so the household passes both income tests." +us,scenario_057,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of a listed SNAP receipt amount as requiring a zero output. The prompt requires calculated eligibility with assumed take-up, and the household qualifies and receives a positive allotment." +us,scenario_057,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,The model omitted the $602.58 monthly SSI amount from SNAP countable income and calculated net income solely from self-employment and the listed disability benefits. This understated the annual household contribution by roughly the same magnitude as its benefit overstatement. +us,scenario_057,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model’s $340 monthly estimate understates countable net income after deductions. The complete calculation yields $1,090.08 of monthly net income and an expected contribution near $327, leaving mostly $219 rather than $340." +us,scenario_057,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model’s $70 monthly benefit implies net income near $1,587 against a $546 maximum. The traced net income is $1,090.08, so the 30% contribution is about $327 and the monthly benefit is mostly $219." +us,scenario_057,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model overstated countable SSI or otherwise derived net monthly income near $1,587 to reach a $70 allotment. The generated SSI and applicable deductions instead produce $1,090.08 of net income and mostly $219 per month." +us,scenario_057,snap,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly asserted that income or assets exceed SNAP limits. Gross income, net income, and $911.76 of assets all pass their respective tests, and SSI receipt also establishes categorical eligibility." +us,scenario_057,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used monthly gross income of about $1,928 by overstating SSI or other unearned income. Correct monthly gross income is $1,472.42 and net income is $1,090.08, producing mostly $219 rather than $71 per month." +us,scenario_057,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,The model omitted generated SSI from countable income and subtracted only $209 for the entire year as the standard deduction instead of applying monthly deductions. It therefore derived neither the correct monthly net income nor the month-specific allotments. +us,scenario_057,snap,inkling,llm_error,taxable_income_or_deductions,False,"The model omitted $602.58 of monthly SSI and based the contribution on only about $488 of monthly net income. Including SSI and applying the full deductions yields $1,090.08 of net income and mostly $219 per month." +us,scenario_057,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value or explanation, so the required output was missing." +us,scenario_057,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model estimated $13,183 of annual SSI, overstating the generated $602.58 monthly amount and producing $1,586.08 of monthly net income. Correct net income is $1,090.08, which yields a substantially larger allotment." +us,scenario_057,snap,minimax-m3,llm_error,asset_resource,False,"The model reversed the asset comparison: $912 is below, not above, an asset limit of about $4,500. The traced bank assets are $911.76, so the household passes the asset test." +us,scenario_057,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly excluded all income of SSI-eligible household members from SNAP countable income. SSI receipt confers categorical eligibility, but the SSI and applicable earned income still enter the allotment calculation, producing $1,090.08 of net monthly income rather than zero." +us,scenario_057,snap,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly denied a positive benefit without applying the eligibility tests. The household passes the net-income, gross-income, asset, categorical-eligibility, and work-requirement tests, after which the allotment formula produces a positive annual amount." +us,scenario_057,spouse_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated Louisiana CHIP as covering low-income adults through a standalone income test. It failed to apply the controlling Medicaid exclusion: the 20-year-old spouse is Medicaid-eligible in the adult category and therefore is not eligible for CHIP, regardless of being below an asserted CHIP income threshold." +us,scenario_057,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply Louisiana's ACA adult expansion pathway and incorrectly demanded additional qualifying details. At age 20 with MAGI of 0.45 times FPL, the spouse qualifies in the MAGI-based ADULT category without another categorical condition." +us,scenario_057,spouse_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly treated pregnancy or parenthood of a minor child as necessary for Louisiana Medicaid expansion. Louisiana's ACA expansion covers low-income adults under 65 regardless of pregnancy or parental status, and the spouse's MAGI of 0.45 times FPL satisfies its financial test." +us,scenario_057,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model incorrectly treated disability alone as sufficient for Medicare eligibility regardless of age. The 20-year-old spouse does not meet the age threshold, and no qualifying under-65 Medicare entitlement is listed." +us,scenario_057,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model inferred both SSDI entitlement and completion of Medicare's 24-month disability waiting period from a generic disability-benefit amount and the instruction that facts remain constant during 2026. That instruction establishes only within-year constancy, not 24 prior months of qualifying SSDI entitlement, and unlisted entitlement history is false." +us,scenario_057,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated being disabled and receiving unspecified disability benefits as PolicyEngine's under-65 Medicare pathway. Neither fact establishes qualifying SSDI entitlement or another under-65 Medicare category, while the spouse is only age 20." +us,scenario_057,spouse_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model misclassified the listed $24 of generic disability benefits as receipt under a Medicare-qualifying disability-beneficiary rule. The prompt supplies no qualifying SSDI entitlement or waiting-period history, so the age-20 spouse is not Medicare eligible." +us,scenario_057,spouse_medicare_eligible,kimi-k3,llm_error,age_disability,False,The model invented a PolicyEngine rule under which the disability boolean itself satisfies under-65 Medicare eligibility. The spouse is below 65 and has no listed qualifying Medicare entitlement pathway. +us,scenario_057,spouse_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented WIC coverage for women up to age 60 and treated disability as a route to postpartum or other categorical eligibility. The spouse is not pregnant, breastfeeding, or postpartum, so satisfying an income limit cannot establish WIC eligibility." +us,scenario_057,spouse_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated being a 20-year-old disabled woman as a qualifying WIC category. Age, sex, and disability do not replace the required pregnancy, breastfeeding, or postpartum status, none of which is present." +us,scenario_057,ssi,claude-fable-5,llm_error,other,False,"The model submitted an unanalyzed placeholder instead of performing the SSI calculation. Applying individual eligibility, the resource test, spousal deeming, and SSI income exclusions yields $7,231 for the head." +us,scenario_057,ssi,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model counted nearly all $10,400 of self-employment income after only a $65-per-month subtraction and therefore eliminated the benefit. SSI earned-income counting and spousal deeming leave only $391.42 of monthly countable income, so the eligible head receives $7,231 annually; the spouse is not SSI-eligible." +us,scenario_057,ssi,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly declared the spouse SSI-eligible and subtracted pooled income from a couple federal benefit rate. Only the head is eligible, and the spouse affects the head through spousal deeming, producing an individual annual benefit of $7,231." +us,scenario_057,ssi,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated both spouses as eligible and calculated a combined couple award. The spouse fails the SSI aged, blind, or disabled eligibility variable, so SSI is calculated only for the head with spousal deeming, yielding $7,231." +us,scenario_057,ssi,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated roughly $9,600 of self-employment income as countable and concluded that it exceeded the benefit rate. The SSI earned-income exclusions and deeming calculation reduce countable income to $391.42 per month, leaving the eligible head a $602.58 monthly payment." +us,scenario_057,ssi,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model assumed the spouse’s generic disabled status made both people SSI-eligible and used the couple federal benefit rate. The spouse is not SSI aged, blind, or disabled under the modeled eligibility variable; only the head receives SSI, with spousal deeming producing $7,231 annually." +us,scenario_057,ssi,claude-sonnet-5,llm_error,other,False,"The reasoning explicitly calculated a positive SSI payment of about $12,590 but the submitted numeric value was 0, so the final output contradicts its own explanation. Its underlying calculation also used an eligible-couple rate even though only the head is SSI-eligible." +us,scenario_057,ssi,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model used an estimated couple federal benefit rate and treated the household as an eligible couple. SSI is payable only to the head, with the spouse included through deeming rather than as a second eligible recipient, yielding $7,231." +us,scenario_057,ssi,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model assumed both spouses were SSI-eligible and calculated the couple maximum less pooled countable income. The spouse is ineligible, and the correct individual head calculation with spousal deeming produces $7,231." +us,scenario_057,ssi,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the head’s self-employment income as categorically disqualifying. SSI exclusions and spousal deeming leave $391.42 in monthly countable income, below the applicable individual benefit amount, so the head receives $7,231 annually." +us,scenario_057,ssi,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model’s reference to a couple maximum shows that it treated both spouses as SSI recipients. Only the head satisfies SSI categorical eligibility, and the individual calculation with spousal deeming yields $7,231." +us,scenario_057,ssi,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model explicitly classified both spouses as eligible and used an annual couple federal benefit rate. The spouse is not SSI-eligible, so only the head’s individual benefit is calculated, with spousal deeming reducing it to $7,231." +us,scenario_057,ssi,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model incorrectly inferred that both adults’ generic disabled flags established SSI eligibility and applied a couple benefit. The spouse fails the modeled SSI aged, blind, or disabled criterion; the head alone receives $7,231 after deeming." +us,scenario_057,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model treated $10,400 of self-employment income as exceeding SSI limits and also relied on SSI receipt not being listed, despite the prompt requesting modeled eligibility and benefit calculation. The blind and disabled head passes the resource test, and SSI income exclusions plus spousal deeming yield a positive $7,231 benefit." +us,scenario_057,ssi,gpt-5.4-mini,llm_error,asset_resource,False,"The model incorrectly stated that $912 of bank assets exceeded the SSI asset limit. The head has $911.76 of countable resources and passes the $2,000 individual limit; after income counting, the head receives $7,231." +us,scenario_057,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of explicitly listed SSI receipt as requiring a zero, instead of calculating the requested benefit under assumed take-up. The blind and disabled head is eligible, passes the resource test, and receives $7,231 after income counting and spousal deeming." +us,scenario_057,ssi,gpt-5.5,llm_error,categorical_eligibility,False,"The model classified both spouses as eligible, applied the $3,000 couple resource limit, and used the eligible-couple benefit rate. Only the head qualifies; the spouse is handled through deeming, and the resulting individual benefit is $7,231." +us,scenario_057,ssi,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model assumed both adults met SSI disability eligibility and used the couple federal benefit rate. The spouse is not SSI-eligible, so the head’s individual award with spousal deeming is $7,231." +us,scenario_057,ssi,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model incorrectly treated both spouses as SSI-eligible and reduced a couple maximum by pooled income. Only the head qualifies, and spousal deeming within the head’s individual calculation yields $7,231." +us,scenario_057,ssi,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model used the SSI couple limit and a combined federal benefit because it declared both adults eligible. The spouse fails SSI categorical eligibility, leaving an individual head benefit of $7,231 after deeming and exclusions." +us,scenario_057,ssi,grok-4.3,llm_error,taxable_income_or_deductions,False,The model asserted that countable self-employment income exceeded the applicable SSI rate without applying the engine’s income exclusions and deeming calculation. Those rules leave $391.42 of monthly countable income and a positive $602.58 monthly payment to the head. +us,scenario_057,ssi,grok-4.5,llm_error,categorical_eligibility,False,"The model treated both spouses as disabled SSI recipients and used a projected couple federal benefit rate. The spouse is not SSI-eligible; only the head receives SSI, and spousal deeming produces $7,231 annually." +us,scenario_057,ssi,grok-build-0.1,llm_error,categorical_eligibility,False,"The model calculated an eligible-couple award from a couple federal benefit rate and joint exclusions. SSI is individual here: the spouse is ineligible and is relevant only through deeming into the head’s $7,231 award." +us,scenario_057,ssi,inkling,llm_error,categorical_eligibility,False,"The model’s use of the federal couple SSI rate shows that it calculated a combined couple award. Only the head is SSI-eligible, and the individual calculation with spousal deeming yields $7,231 rather than $13,437." +us,scenario_057,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required output contract." +us,scenario_057,ssi,kimi-k3,llm_error,categorical_eligibility,False,"The model assumed both spouses were eligible, used the $3,000 couple resource limit, and subtracted income from the couple benefit rate. The spouse is not SSI-eligible; the head passes the individual resource test and receives $7,231 after spousal deeming." +us,scenario_057,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model inferred SSI nonreceipt from the small listed disability-benefit amounts instead of applying the blind or disabled eligibility pathway and assumed program take-up. The head satisfies SSI categorical and resource eligibility, and the income calculation produces $7,231." +us,scenario_057,ssi,qwen-3.7-max,llm_error,period_annualization,False,"The model treated the prompt’s annual $14 and $24 disability-benefit amounts as monthly income, repeatedly changed projected benefit rates, and alternated between individual and couple calculations. The amounts are annual, only the head is eligible, and the consistent individual calculation with spousal deeming yields $7,231." +us,scenario_057,ssi,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that the listed income and resources produced no payment without applying the SSI exclusions and deeming computation. The head passes the resource test, and $391.42 of monthly countable income leaves a $602.58 monthly SSI benefit." +us,scenario_057,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied obsolete Louisiana rules: two $4,500 personal exemptions and a 1.75% first tax bracket. Under the 2026 married-filing-jointly deduction and rate structure, the household’s approximately $9,665 of adjusted income produces zero Louisiana taxable income and therefore $0 tax before refundable credits." +us,scenario_057,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value for state_income_tax_before_refundable_credits, violating the required output contract. The required computation yields $0 because the household has no positive Louisiana taxable income under the 2026 married-filing-jointly rules." +us,scenario_057,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model identified Louisiana’s 5% EITC and the childless federal EITC pathway but then discarded both without completing the calculation. The household’s self-employment earnings produce a $664 federal EITC, yielding a $33.20 refundable Louisiana EITC." +us,scenario_057,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model omitted Louisiana’s refundable EITC and incorrectly treated the absence of dependents as eliminating refundable credits. Childless joint filers can receive the federal EITC, and Louisiana refunds 5% of the resulting $664 credit." +us,scenario_057,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly required a qualifying child or related expense for Louisiana’s EITC. The couple qualifies for the childless federal EITC on the head’s self-employment earnings, producing a $33.20 Louisiana credit." +us,scenario_057,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model recognized the 5% Louisiana EITC but incorrectly rounded a nonzero childless credit to zero. The federal EITC is $664, so its 5% state match is exactly $33.20." +us,scenario_057,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly classified Louisiana’s 5% EITC as nonrefundable or otherwise limited out. It is refundable here, and the $664 federal EITC generates $33.20 of state refundable credit." +us,scenario_057,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana has no state EITC or applicable refundable credit in 2026. Louisiana refunds an EITC equal to 5% of the federal $664 credit, producing $33.20." +us,scenario_057,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly denied that Louisiana offers an applicable refundable EITC and treated dependents as necessary. Louisiana’s refundable credit equals 5% of the childless couple’s $664 federal EITC, or $33.20." +us,scenario_057,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model applied the correct 5% Louisiana rate to an understated federal EITC of $649. The federal computation yields $664, making the state credit $33.20 rather than $32.45." +us,scenario_057,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,The model applied the correct 5% state rate but understated the federal EITC as $630. Using the computed federal EITC of $664 yields a Louisiana credit of $33.20. +us,scenario_057,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The zero answer omits the refundable Louisiana EITC generated by the head’s self-employment earnings. The $664 federal EITC produces a $33.20 state credit at Louisiana’s 5% rate. +us,scenario_057,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly declared the household ineligible for every Louisiana refundable credit. The couple qualifies for a $664 federal EITC, which generates a refundable Louisiana EITC of $33.20." +us,scenario_057,state_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model incorrectly used the spouse’s age of 20 to disqualify the joint return from the childless EITC. The head is age 43 and satisfies the applicable age condition, so the $664 federal EITC generates a $33.20 Louisiana credit." +us,scenario_057,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model omitted the Louisiana refundable EITC triggered by the household’s earned self-employment income. The federal EITC is $664, and Louisiana refunds 5% of it, or $33.20." +us,scenario_057,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the listed facts as triggering no refundable state credit, overlooking Louisiana’s federal-EITC match. The household’s $664 federal EITC produces $33.20 in Louisiana refundable credits." +us,scenario_057,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly required children for Louisiana’s refundable EITC. This two-adult joint return qualifies through the childless federal EITC pathway, and 5% of its $664 federal credit is $33.20." +us,scenario_057,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model failed to recognize that self-employment earnings trigger the childless federal EITC and its Louisiana match. The resulting $664 federal credit produces $33.20 of refundable state credit. +us,scenario_057,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model’s blanket ineligibility conclusion omits Louisiana’s refundable EITC. The household receives a $664 federal EITC, of which Louisiana refunds 5%, yielding $33.20." +us,scenario_057,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana has no refundable credit applicable to this household. Louisiana’s refundable EITC equals 5% of the household’s $664 federal EITC, or $33.20." +us,scenario_057,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,The model omitted Louisiana’s refundable 5% match of the federal EITC. Applying that rule to the household’s $664 federal EITC gives $33.20. +us,scenario_057,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model incorrectly computed the federal EITC as zero despite the head’s qualifying self-employment earnings and the childless joint-filer pathway. The federal EITC is $664, so Louisiana’s 5% refundable match is $33.20." us,scenario_057,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, violating the required output contract." -us,scenario_057,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model doubled Louisiana's EITC percentage by using 10% instead of 5%. Applying the 5% rate to the correctly identified $664 federal EITC gives $33.20. -us,scenario_057,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana has no individual income tax. Louisiana imposes an individual income tax and provides a refundable EITC equal to 5% of the federal EITC, yielding $33.20 here." -us,scenario_057,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana has no state individual income tax and therefore no refundable credits. Louisiana's refundable EITC equals 5% of the household's $664 federal EITC, or $33.20." -us,scenario_059,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security and then left positive taxable income after the senior married-filing-jointly standard deduction. Its own calculation produces taxable income solely from a rough taxable-Social-Security estimate, but the correct computation applies the full deductions to the household's AGI and reduces taxable income to zero." -us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a sharply understated married-filing-jointly senior standard deduction of $18,100. That deduction error converts the correct AGI with $6,129.20 of taxable Social Security into artificial taxable income, while the correct senior MFJ deduction treatment eliminates taxable income." -us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model mixed a post-TCJA standard deduction with personal exemptions and senior additions that still totaled only $29,125. The correct deduction treatment for this 2026 married senior couple is large enough to reduce taxable income to zero, so the 10% tax on $2,604.20 is an artifact of using the wrong deduction structure." -us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated 85% of total Social Security as taxable instead of applying the Social Security benefits worksheet result reflected in the AGI trace. It then applied a nonrefundable elderly/disabled credit to a liability that should never arise, because the correct taxable-income calculation is zero before any such credit is needed." -us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used a pre-TCJA-style deduction and personal exemption stack of about $29,600 against AGI. That left $2,129 of artificial taxable income, while the correct 2026 deduction calculation for the married senior household reduces taxable income to zero." -us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an incorrect hybrid of a small standard deduction, senior additions, and personal exemptions totaling about $28,400. The correct deduction treatment for this married couple over age 65 fully offsets the $31,729 AGI, so there is no taxable income to tax at 10%." -us,scenario_059,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at $12,129 and used a pre-TCJA deduction-plus-personal-exemption framework that still left positive taxable income. The correct computation uses the trace's lower taxable Social Security amount and the applicable senior married-filing-jointly deductions, reducing taxable income to zero." -us,scenario_059,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly made $16,876 of Social Security taxable by applying an 85% shortcut and also misstated the senior additional standard deduction calculation. Those errors created $10,176 of taxable income, while the correct Social Security inclusion and deduction calculation leaves taxable income at zero." -us,scenario_059,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's reasoning correctly excluded Social Security retirement income and IRA distributions from employee payroll tax and stated that payroll tax is 0. Its submitted value of 1937 contradicts that explanation, so the failure is an answer-contract/value entry error rather than a payroll-tax rule error." -us,scenario_060,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model computed the childless EITC from wages alone as $3,000 × 7.65% and ignored the household's full income facts when deciding refundable-credit eligibility. It also applied a childless EITC pathway for a 70-year-old in 2026 after acknowledging the ARPA age expansion expired, so it created an EITC where the engine returns none and no other refundable credit applies." -us,scenario_060,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly placed the person in a Texas aged/blind/disabled Medicaid category based on age, low bank assets, and an SSI-related income test. PolicyEngine's category computation assigns NONE: the person receives no SSI and does not qualify through any Texas Medicaid pathway, so the MAGI level and excluded veterans-benefit reasoning never establish eligibility." -us,scenario_060,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used a shortcut that equates old age, low income, and minimal resources with Texas Medicaid eligibility. The correct computation first requires a qualifying Medicaid category, and PolicyEngine found no category for this 70-year-old head, so low income and resources do not produce eligibility." -us,scenario_060,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly computed $186.00 of Social Security tax and $43.50 of Medicare tax, totaling $229.50, but then submitted $232.50 without any additional tax component or valid rounding step. It introduced an unexplained $3 error after completing the correct calculation." -us,scenario_060,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model applied the combined employer-and-employee Medicare rate of 2.9% to the employee-only payroll-tax output. The employee Medicare rate is 1.45%, producing $43.50 rather than $87.00 and a total payroll tax of $229.50." -us,scenario_060,snap,claude-opus-4.7,llm_error,other,False,"The model correctly counted the $45,600 veterans benefits and computed net income far above the SNAP net income limit, then ignored its own eligibility conclusion and submitted a positive benefit. Its $2,496 answer is inconsistent with the stated derivation that the household is income-ineligible and reflects an abandoned minimum-or-partial-benefit shortcut instead of applying the final eligibility screen." -us,scenario_060,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model excluded the $45,600 veterans benefits from SNAP countable income, reducing gross monthly income to $1,431 instead of counting total annual income of $62,775. That exclusion let rent and other deductions drive net income to zero and produced a maximum-allotment benefit, but counting veterans benefits leaves the household over the income limit with no SNAP benefit." -us,scenario_060,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model counted only wages, Social Security, and interest as SNAP income, using $1,431.25 per month and omitting the $45,600 in veterans benefits. With the veterans benefits included, the medical and shelter deductions do not reduce net income to zero, so the household fails the SNAP income test instead of receiving the maximum allotment." -us,scenario_062,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model double-counted the $28,800 survivor-benefits field as ordinary taxable income in addition to Social Security survivor benefits, which drove its AGI to $63,245 and made 85% of Social Security taxable. It also incorrectly used qualifying-surviving-spouse filing status without a qualifying child; the filer is single." -us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model double-counted $28,800 of survivor benefits as taxable income, producing the erroneous $63,245 AGI and 85% Social Security inclusion. It also omitted the 2026 senior deduction, so it understated the deductions that eliminate taxable income." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA individual provisions expired for 2026 and substituted restored personal exemptions, pre-TCJA rates, and a much smaller deduction package. It also double-counted the $28,800 survivor-benefits field as taxable income, inflating AGI and taxable Social Security." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated all $21,312 of Social Security as AGI, added the overlapping $28,800 survivor-benefits amount, and then incorrectly subtracted farm-rent income. It omitted the senior deduction and its stated $29,449 taxable income cannot arithmetically produce only $178 of federal tax." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the separate $28,800 survivor-benefits amount and reached roughly $17,323 of AGI, but then applied an obsolete post-sunset deduction regime. The 2026 standard deduction, age-65 addition, and senior deduction together exceed that AGI, leaving zero taxable income rather than $1,523." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model double-counted the $28,800 survivor-benefits field as taxable income and therefore forced 85% of Social Security into AGI. It also replaced the applicable 2026 standard, age-based, and senior deductions with an unsupported $15,400 total." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model double-counted the $28,800 survivor-benefits amount and used an inapplicable post-TCJA-sunset system with a personal exemption and 15% and 25% brackets. It omitted the 2026 senior deduction that, together with the standard and age-based deductions, eliminates taxable income." -us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its answer implies that it counted the separate $28,800 survivor-benefits field as taxable income alongside Social Security, pension, interest, and farm rent. Properly avoiding that double count and applying the standard, age-based, and senior deductions leaves no taxable income." -us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model explicitly included taxable survivor income in addition to taxable Social Security, double-counting the survivor-benefit inputs and inflating AGI. After the survivor-benefit overlap is removed, the deductions it identified fully absorb taxable income." -us,scenario_062,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model double-counted $28,800 of survivor benefits and assumed the TCJA rules expired, leading it to use a restored personal exemption and pre-TCJA rates. It omitted the 2026 senior deduction and therefore created taxable income where none remains." -us,scenario_062,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model double-counted the $28,800 survivor-benefits field, which inflated provisional income enough to classify 85% of Social Security as taxable. It then applied an inapplicable personal exemption and pre-TCJA 15% bracket while omitting the 2026 senior deduction." -us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response failed the required output contract." -us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly avoided treating the $28,800 survivor-benefits field as additional taxable AGI and reached about $17,323 of AGI. It nevertheless used a restored-exemption deduction estimate and omitted the 2026 senior deduction, which reduces taxable income to zero." -us,scenario_062,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted the $28,800 survivor-benefits amount as taxable income, driving AGI to $63,245 and making 85% of Social Security taxable. It also omitted the senior deduction and applied an understated deduction total, producing $46,695 of spurious taxable income." +us,scenario_057,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model doubled Louisiana’s EITC percentage by applying a 10% rate. The applicable rate is 5%, so the $664 federal EITC produces $33.20 rather than $66.40." +us,scenario_057,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana has no individual income tax. Louisiana imposes an individual income tax and provides a refundable EITC equal to 5% of the federal $664 credit, yielding $33.20." +us,scenario_057,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana has no state individual income tax and therefore no refundable credits. Louisiana’s refundable EITC is 5% of the household’s $664 federal EITC, producing $33.20." +us,scenario_057,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model overlooked the refundable Louisiana EITC available through the childless federal EITC pathway. The household’s $664 federal EITC generates a $33.20 state credit at the 5% rate. +us,scenario_059,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security instead of applying the statutory married-filing-jointly Social Security worksheet, which yields $6,129.20 and joint AGI of $31,729.20. It also omitted the full 2026 senior deductions that reduce taxable income to zero, and its unexplained jump from a stated $540 calculation to $1,177 is arithmetically invalid." +us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model correctly derived $6,129.20 of taxable Social Security and $31,729.20 of AGI, but used only an estimated $18,100 deduction for a joint return with two elderly spouses. Applying the full 2026 standard and senior deductions reduces taxable income to zero, not $13,629." +us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly identified AGI of $31,729.20 but substituted an incorrect mixture of a reduced standard deduction, personal exemptions, and undersized senior deductions totaling $29,125. The full 2026 deductions for both spouses over 65 exceed AGI and eliminate taxable income." +us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated 85% of Social Security as automatically taxable; the married-filing-jointly worksheet yields only $6,129.20 of taxable benefits and AGI of $31,729.20. It then omitted the full 2026 senior deductions and invented a usable elderly credit rather than first reducing taxable income to zero." +us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model correctly used AGI of about $31,729 but deducted only a rough $29,600 based on a pre-TCJA standard-deduction-and-exemption framework. The applicable 2026 standard and senior deductions exceed AGI, leaving no taxable income and no tentative tax." +us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model correctly derived approximately $31,729 of AGI but applied an incorrect $15,300 joint standard deduction and only $3,100 of elderly additions, supplemented by personal exemptions. The full 2026 deductions for two spouses over 65 reduce taxable income to zero." +us,scenario_059,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at $12,129; the joint-return worksheet yields $6,129.20 and AGI of $31,729.20. It also used an obsolete pre-TCJA deduction-and-personal-exemption framework and omitted the full 2026 senior deductions that eliminate taxable income." +us,scenario_059,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model both misstated gross Social Security as $37,504 and treated $16,876 as taxable instead of applying the joint Social Security worksheet, which yields $6,129.20. Its deduction calculation also omitted the full 2026 senior deductions, so it manufactured $10,176 of taxable income where the correct deduction step leaves zero." +us,scenario_059,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly used the 85% cap as the taxable Social Security amount rather than applying the married-filing-jointly worksheet, overstating AGI from $31,729.20 to $57,138. It also treated a $5,000 mortgage balance as a residential clean-energy credit, then added that supposed $6,000 credit to liability instead of subtracting it; a mortgage balance creates no such credit, and the proper deductions already reduce taxable income to zero." +us,scenario_059,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly stated that the household had no payroll-taxable wages and therefore owed zero employee payroll tax, but then submitted $1,937 without any supporting computation. Its numeric output directly contradicts its own stated payroll-tax-base calculation." +us,scenario_060,federal_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model correctly noted that the age-65 expansion applied only in 2021 but then applied the childless EITC phase-in rate anyway. At age 70, the head fails the under-65 age requirement for a taxpayer without qualifying children, so the $3,000 of wages generates no EITC rather than $229.50." +us,scenario_060,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly excluded the $45,600 of veterans benefits when asserting that the head passed Texas's aged Medicaid income test. The head does not satisfy the aged pathway or any other Medicaid category, so low bank assets and MAGI of 1.08 times FPL do not establish eligibility." +us,scenario_060,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age, purportedly low income, and minimal resources as sufficient for Texas Medicaid without applying a qualifying categorical pathway. It also overlooked the listed $45,600 in veterans benefits; the head qualifies through no pathway and is assigned Medicaid category NONE." +us,scenario_060,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly computed $186.00 of employee Social Security tax plus $43.50 of employee Medicare tax and explicitly obtained $229.50, then submitted $232.50 without any additional tax component or valid rounding step. It introduced an unsupported $3.00 arithmetic discrepancy after completing the correct derivation." +us,scenario_060,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model applied the combined employer-and-employee Medicare rate of 2.9% to wages, producing $87, instead of the employee rate of 1.45%, which produces $43.50. Because the requested output excludes employer payroll taxes, employee FICA totals $229.50, not $273.00." +us,scenario_060,snap,claude-opus-4.7,llm_error,other,False,"The model correctly counted the $45,600 of veterans benefits and calculated net income of about $3,994 per month, which exceeds the one-person net-income limit. It then discarded its own eligibility conclusion and submitted $2,496 without any computation supporting that amount; an income-ineligible household receives $0, not a minimum or partial allotment." +us,scenario_060,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $45,600 of veterans benefits from SNAP countable unearned income. Including those benefits prevents the rent deduction from reducing net income to zero and leaves the household above the one-person net-income eligibility limit." +us,scenario_060,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model’s $1,431.25 monthly gross-income figure omits the $3,800 monthly veterans benefit from SNAP countable income. That omission lets the medical and shelter deductions erase all net income; with veterans benefits included, countable net income remains above the eligibility threshold and SNAP equals $0." +us,scenario_062,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $28,800 generic survivor-benefits amount to taxable AGI and treated surviving-spouse status as qualifying-widow filing status without the required dependent-child pathway. The actual single-filer calculation excludes that amount from taxable AGI, and the standard, age-based, and senior deductions eliminate the remaining taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $28,800 survivor-benefits amount as taxable income, which also drove Social Security taxation to 85% and inflated AGI to $63,245. It then omitted the 2026 senior deduction; excluding the nontaxable survivor-benefits amount and applying all age-related deductions reduces taxable income to zero." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model taxed the $28,800 survivor-benefits amount and assumed a TCJA sunset with restored personal exemptions and pre-TCJA rates. Under 2026 current law, that survivor-benefits amount is not taxable AGI and the standard, age-65, and senior deductions eliminate the remaining taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model constructed AGI by counting the full Social Security benefit and the $28,800 survivor-benefits amount while subtracting farm rent, then reported a tax figure inconsistent with its own stated taxable income. Federal AGI instead includes taxable farm rent and only the computed taxable Social Security portion, excludes the generic survivor-benefits amount, and is fully offset by the applicable deductions." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model correctly arrived near $17,323 of AGI but applied an obsolete TCJA-sunset deduction regime and omitted the 2026 senior deduction. The single standard deduction, age-65 additional deduction, and senior deduction together reduce taxable income to zero." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $28,800 survivor-benefits amount in taxable AGI and treated 85% of Social Security as taxable as a consequence. It also failed to apply the full 2026 senior deduction, which together with the single standard and age-based deductions eliminates the correctly computed taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model taxed the $28,800 survivor-benefits amount, forcing 85% of Social Security into AGI, and used a post-TCJA-sunset personal-exemption regime. The 2026 current-law calculation excludes the generic survivor-benefits amount and applies the standard, age-65, and senior deductions, leaving zero taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's explanation explicitly includes the survivor-benefits amount in federal taxable income even though that generic benefit is not a taxable AGI item. Removing it changes the taxable Social Security calculation, and the standard, age-based, and senior deductions then eliminate all taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model explicitly treated the $28,800 survivor-benefits amount as taxable, inflating both AGI and the taxable share of Social Security. With that amount excluded and the stated single, age-based, senior, and applicable business-income deductions applied, taxable income is zero." +us,scenario_062,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $28,800 survivor-benefits amount in AGI and applied an assumed post-TCJA-sunset exemption-and-rate structure. Current 2026 deductions include the senior deduction, and after excluding the nontaxable survivor-benefits amount they reduce taxable income to zero." +us,scenario_062,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly taxed the $28,800 survivor-benefits amount, thereby overstating provisional income and taxing 85% of Social Security. It also substituted a restored personal exemption for the 2026 senior deduction; the correct income base and deductions yield no taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model added the $28,800 survivor-benefits amount to AGI and consequently treated $18,115 of Social Security as taxable. It omitted the senior deduction from its listed deductions; excluding the generic survivor benefit and applying the full age-related deductions reduces taxable income to zero." +us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for the requested variable, so it failed the required structured-output contract." +us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the generic survivor-benefits amount and arrived at approximately $17,323 of AGI, but it used a restored-personal-exemption framework and omitted the 2026 senior deduction. Applying the single standard deduction, age-65 addition, and senior deduction leaves zero taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model ultimately treated the $28,800 survivor-benefits amount as taxable, pushed Social Security to the 85% taxable maximum, and omitted the 2026 senior deduction. The survivor-benefits amount is excluded from taxable AGI, and the standard, age-65, and senior deductions eliminate the remaining taxable income." us,scenario_062,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_062,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a Florida senior pathway through medically needy, spousal protections, or unspecified exclusions even though the person has no qualifying Medicaid category. It also acknowledged bank assets far above a typical SSI-related resource limit but then overrode that disqualifying fact without applying a concrete PolicyEngine eligibility pathway." -us,scenario_062,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded Social Security survivor benefits from countable income and treated only pension, interest, and farm rent as the aged/disabled income base. PolicyEngine does not place this person in an aged/disabled Medicaid category, and SSI received is zero, so the model's SSI-related income shortcut creates eligibility that the engine does not recognize." -us,scenario_062,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated Florida Medically Needy and Medicare Savings Program concepts as equivalent to the requested Medicaid eligibility output. PolicyEngine's Medicaid eligibility result requires an evaluated Medicaid category, and this person has medicaid_category = NONE with MAGI income at 2.36 FPL, so age, disability, resources, and health expenses do not make the head eligible." -us,scenario_062,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model collapsed elderly and disabled status into Medicaid eligibility without applying Florida's categorical pathway screen. PolicyEngine finds no Medicaid category and no SSI receipt, so the submitted Yes is consistent with assuming demographics alone establish eligibility." -us,scenario_062,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model inferred SSI-linked Medicaid or Medicare Savings Program eligibility from age, disability, Medicare likelihood, and health expenses instead of applying the engine's Medicaid category result. PolicyEngine records SSI received as zero and medicaid_category = NONE, so those facts do not open a Medicaid eligibility pathway for this Florida household." +us,scenario_062,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model acknowledged that the bank assets exceed the SSI-related resource limit, then invented medically needy and spousal-protection eligibility without facts establishing either pathway. Surviving-spouse status does not supply a community spouse or override the absence of a qualifying Medicaid category." +us,scenario_062,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the Social Security survivor benefit from SSI-related countable income and then characterized $16,330 of its remaining annual income as below the aged/disabled standard. The head receives no SSI and qualifies under no aged/disabled Medicaid category." +us,scenario_062,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model substituted speculative medically needy spend-down or Medicare Savings Program eligibility for the Medicaid category tested by this output. Disability, health expenses, and Medicare eligibility do not establish either pathway, and the engine assigns the head no Medicaid eligibility category." +us,scenario_062,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The answer treated old age and disability as sufficient categorical eligibility and asserted that income and assets were low enough without applying a Florida pathway or its limits. The head has no qualifying Medicaid category, receives no SSI, and has MAGI income of 2.36 times FPL." +us,scenario_062,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model conflated Medicare Savings Program or SSI-linked coverage with full Medicaid eligibility and assumed deductions from medical expenses would create eligibility. The head receives zero SSI and qualifies under no Medicaid category, so age, disability, Medicare eligibility, and health expenses do not produce Medicaid eligibility." us,scenario_062,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_062,self_employment_tax,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated the $4,793 of farm rent income as net self-employment earnings and applied the standard 92.35% self-employment earnings adjustment and 15.3% SECA rate to it. PolicyEngine's self_employment_tax base contains no self-employment income for this household, so the model taxed a rent-income item that does not enter the self-employment tax base." -us,scenario_064,child1_chip_eligible,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model tested Child 1's own reported income instead of income attributed through the child's CHIP MAGI eligibility unit. The applicable household income is approximately $98,810, which exceeds Wisconsin's CHIP limit for that unit." -us,scenario_064,child1_chip_eligible,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model applied the BadgerCare Plus percentage to an oversized household unit and therefore classified the income as below 300% FPL. For Child 1's applicable MAGI unit, approximately $98,810 exceeds the CHIP income ceiling." -us,scenario_064,child1_chip_eligible,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model compared income with 306% FPL using the wrong household-size basis. Child 1 is age-eligible, but the applicable MAGI unit's approximately $98,810 income is above Wisconsin's CHIP threshold." -us,scenario_064,child1_chip_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model incorrectly placed the child between the Medicaid ceiling and Wisconsin's upper CHIP income band. The trace places approximately $98,810 above the applicable CHIP ceiling, so there is no CHIP eligibility band available to Child 1." -us,scenario_064,child1_chip_eligible,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model divided its MAGI estimate by the FPL for all five co-residents, producing 273% FPL and a false income pass. CHIP instead uses Child 1's applicable MAGI eligibility unit, under which the traced income exceeds Wisconsin's limit; its ESI-affordability discussion never cures that failed income test." -us,scenario_064,child1_chip_eligible,kimi-k2.6,llm_error,household_unit_or_filing_status,False,"The model used a five-person FPL threshold merely because five people live together and then treated losses and contributions as sufficient to fall below 300% FPL. Wisconsin CHIP applies the child's MAGI eligibility unit, for which approximately $98,810 exceeds the income threshold despite Child 1 satisfying the under-19 age rule." -us,scenario_064,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model asserted eligibility under a 300% FPL child limit without calculating that the household’s MAGI equals 322% FPL. Because 322% exceeds that limit and no other active category applies, Child 1 is not Medicaid eligible." -us,scenario_064,child1_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model overstated the MAGI reduction from farm losses, capital losses, and unspecified deductions and incorrectly placed the household below the child threshold. The trace yields MAGI of 322% FPL, above the asserted 301% limit, and no categorical pathway applies." -us,scenario_064,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model abandoned its own calculation showing household income above 300% FPL and substituted a child-only income or simplified-income test. As a tax-unit dependent, Child 1 is evaluated with the applicable MAGI household income, which is 322% FPL; the child’s zero personal income does not create a separate eligibility pathway." -us,scenario_064,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model used a six-person household and treated the full long-term capital loss and other losses as reducing Medicaid MAGI enough to fall below roughly 300% FPL. The applicable computation produces 322% FPL, and the child qualifies through no non-MAGI category." -us,scenario_064,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model computed MAGI as $114,422 and compared it with an inflated 306%-of-FPL threshold plus an additional 5% disregard. The engine’s applicable MAGI comparison is 322% FPL and does not yield an active child Medicaid category, so medicaid_category is NONE." -us,scenario_064,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied neither the required numeric output nor an explanation for child1_medicaid_eligible. It therefore failed the output contract rather than completing the Medicaid eligibility computation. -us,scenario_064,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 12 alone as a WIC categorical eligibility trigger and ignored the WIC income screen. It also used the wrong child age pathway: WIC covers infants and children under age 5, not all minors, and this household's income exceeds Wisconsin's WIC limit." +us,scenario_062,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly classified the $4,793 of farm rent income as self-employment income. Farm rent does not create net earnings from self-employment here, so the correct tax base is zero rather than $4,793 and self-employment tax is $0." +us,scenario_064,child1_chip_eligible,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model tested Child 1's personal income of zero instead of the income of the child's applicable household. Applying the household's approximately $98,810 income causes the child to fail Wisconsin CHIP's income test." +us,scenario_064,child1_chip_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly treated Wisconsin CHIP as available up to 300% of the applicable federal poverty level and placed the family below that ceiling. The applicable CHIP income test excludes this household at approximately $98,810." +us,scenario_064,child1_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a 306% FPL ceiling as the controlling Wisconsin CHIP threshold and incorrectly classified the household as below it. Under the modeled Wisconsin CHIP income test, approximately $98,810 exceeds the applicable threshold." +us,scenario_064,child1_chip_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model invented an upper CHIP income band that it placed above the household's income. Child 1 is under 19 and not Medicaid-eligible, but the separate Wisconsin CHIP income test also fails at approximately $98,810." +us,scenario_064,child1_chip_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model misapplied a 300% FPL ceiling and then added an employer-coverage affordability exception that does not cure the failed modeled income test. At approximately $98,810, the household exceeds the applicable Wisconsin CHIP threshold regardless of its premium-to-income calculation." +us,scenario_064,child1_chip_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model incorrectly used a 300% FPL threshold for a five-person household and concluded that deductions and losses brought MAGI below it. The applicable Wisconsin CHIP income test places the household's approximately $98,810 income above the eligibility threshold." +us,scenario_064,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,The model asserted eligibility under a 300% FPL child limit without calculating that the household is at 322% FPL. It also invoked unspecified alternative pathways even though Child 1 qualifies through none of them and has Medicaid category NONE. +us,scenario_064,child1_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model overstated the MAGI reduction from the farm loss, capital loss, and unspecified other deductions, concluding that income fell below its 301% FPL threshold. The resulting MAGI level is 322% FPL, so the child fails the applicable income test and has no alternative Medicaid category." +us,scenario_064,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model replaced the child’s MAGI household-income test with a nonexistent separate child-only test based on the child’s $0 personal income. The applicable household MAGI is 322% FPL, above the Wisconsin child Medicaid limit, and no other categorical pathway applies." +us,scenario_064,child1_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly treated the household as six people even though the prompt lists five, inflating the FPL denominator and the purported 300% threshold. It also overstated the effect of farm and capital losses; the Medicaid MAGI calculation yields 322% FPL and no qualifying category." +us,scenario_064,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model applied an incorrect 306% FPL Medicaid threshold and then added a 5% disregard on top of that threshold to force eligibility. PolicyEngine’s applicable pathway evaluation places the household at 322% FPL, above the active child Medicaid limit, with Medicaid category NONE." +us,scenario_064,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child1_medicaid_eligible, so it failed the required output contract." +us,scenario_064,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 12 as within WIC's eligible child age range. WIC covers children only through age four, so Child 1 fails categorical eligibility regardless of income; the household also exceeds the income limit." us,scenario_064,dependent1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_064,dependent1_medicaid_eligible,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model attributed the head and spouse's tax-unit MAGI to dependent1. PolicyEngine assigns dependent1 a MAGI income level of 0.00 times FPL in the ADULT category, which satisfies Wisconsin's adult-expansion income test." -us,scenario_064,dependent1_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied an asset test using assets owned by the household to deny coverage. Dependent1 qualifies through the MAGI-based ADULT category at 0.00 times FPL, a pathway that does not use those household assets as a disqualifying resource test." -us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model substituted the household's aggregate income for dependent1's Medicaid MAGI. The applicable ADULT-category calculation gives dependent1 an income level of 0.00 times FPL, below the expansion threshold." -us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,The model treated the head's substantial income as dependent1's income under both MAGI and disability pathways. PolicyEngine instead qualifies dependent1 through the MAGI-based ADULT category using dependent1's 0.00-times-FPL income level. -us,scenario_064,dependent1_medicaid_eligible,claude-opus-5,llm_error,household_unit_or_filing_status,False,The model denied eligibility from aggregate household income without applying dependent1's person-specific ADULT-category MAGI calculation. That calculation yields 0.00 times FPL and passes Wisconsin's expansion-adult limit. -us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model calculated Medicaid eligibility from the listed household members' combined income and compared that total with 138% FPL. PolicyEngine's ADULT-category calculation places dependent1 at 0.00 times FPL, so the expansion income test is satisfied." -us,scenario_064,dependent1_medicaid_eligible,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model used the household's income to conclude that dependent1 exceeded Wisconsin's adult limit. The relevant MAGI income level for dependent1 is 0.00 times FPL, which passes the ADULT-category threshold." -us,scenario_064,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model's income-based denial implies that it assigned the household's earnings to dependent1. PolicyEngine calculates dependent1's ADULT-category MAGI income level as 0.00 times FPL, below the applicable limit." -us,scenario_064,dependent1_medicaid_eligible,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,The model compared aggregate household income with the adult Medicaid threshold instead of using dependent1's modeled MAGI. Dependent1's ADULT-category income level is 0.00 times FPL and therefore meets the expansion test. -us,scenario_064,dependent1_medicaid_eligible,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model incorrectly included the head's wages, partnership income, and retirement distributions in dependent1's Medicaid MAGI and also treated employer-sponsored insurance as adverse to eligibility. PolicyEngine evaluates dependent1 at 0.00 times FPL in the ADULT category, and existing employer coverage does not defeat that eligibility." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model used the shortcut that an adult living in a high-income household is ineligible. The applicable person-level ADULT-category calculation gives dependent1 MAGI of 0.00 times FPL, which is below Wisconsin's expansion threshold." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model treated missing disability-program details and the household's income as grounds for denial. Disability qualification is unnecessary because dependent1 passes the separate MAGI-based ADULT pathway at 0.00 times FPL. -us,scenario_064,dependent1_medicaid_eligible,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model assigned the family's annual income to dependent1 merely because dependent1 is a tax dependent. PolicyEngine's Medicaid calculation gives dependent1 an ADULT-category MAGI income level of 0.00 times FPL, which passes the income limit." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-luna,llm_error,asset_resource,False,"The model invoked income and resource rules without applying the qualifying MAGI adult pathway. Dependent1 qualifies in the ADULT category at 0.00 times FPL, where the listed household resources do not create the asserted disqualification." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-terra,llm_error,asset_resource,False,The model treated the available household income and assets as disqualifying despite dependent1's separate MAGI result. The ADULT-category calculation is 0.00 times FPL and does not deny this MAGI eligibility based on the listed assets. -us,scenario_064,dependent1_medicaid_eligible,grok-4.3,llm_error,household_unit_or_filing_status,False,The model's bare threshold denial implies that it compared household income rather than dependent1's Medicaid MAGI with the limit. Dependent1 is evaluated at 0.00 times FPL in the ADULT category and passes the expansion threshold. -us,scenario_064,dependent1_medicaid_eligible,grok-4.5,llm_error,categorical_eligibility,False,"The model focused on failure of a non-MAGI aged, blind, or disabled resource test and also asserted failure of the MAGI adult limit. PolicyEngine qualifies dependent1 through the separate ADULT MAGI pathway at 0.00 times FPL, so failure of the ABD pathway does not control the result." -us,scenario_064,dependent1_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model calculated household MAGI near 273% FPL and applied an ABD asset test as though both barred all Medicaid pathways. PolicyEngine evaluates the distinct ADULT MAGI pathway for dependent1 at 0.00 times FPL, which establishes eligibility regardless of the asserted ABD failure." -us,scenario_064,dependent1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer or explanation for dependent1_medicaid_eligible. It therefore failed the required output contract rather than completing the ADULT-category eligibility calculation. -us,scenario_064,dependent1_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The unexplained denial omits the qualifying ADULT expansion pathway. PolicyEngine calculates dependent1's MAGI income level as 0.00 times FPL, which is below the applicable threshold." -us,scenario_064,dependent1_medicaid_eligible,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model included the head's wages and other income in dependent1's Medicaid MAGI and compared the resulting household total with an adult limit. PolicyEngine instead gives dependent1 an ADULT-category MAGI income level of 0.00 times FPL, satisfying Wisconsin's expansion threshold." -us,scenario_064,dependent1_medicare_eligible,claude-fable-5,llm_error,health_coverage,False,The model converted generic disability status into SSDI-based Medicare eligibility without any fact showing SSDI entitlement or completion of the Medicare disability waiting period. A 27-year-old with only `is disabled` listed does not satisfy the Medicare eligibility condition in this benchmark. -us,scenario_064,dependent1_medicare_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model applied an overbroad rule that disability alone qualifies someone for Medicare regardless of age. The correct computation requires a specific Medicare disability pathway, such as SSDI entitlement after the waiting period, ESRD, or ALS, none of which is listed for Dependent 1." -us,scenario_064,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model correctly named the SSDI waiting-period pathway but then assumed the listed disability status implied SSDI receipt and a completed 24-month waiting period. The prompt states unlisted facts are false, so Dependent 1 has no SSDI entitlement, ESRD, or ALS fact to override being under age 65." -us,scenario_064,dependent1_medicare_eligible,gpt-5.5,llm_error,health_coverage,False,"The model treated `is disabled` as a direct Medicare eligibility flag for an under-65 person. The correct derivation keeps disability separate from Medicare coverage unless a qualifying Medicare pathway is explicitly present, and Dependent 1 has no such listed pathway." -us,scenario_064,dependent2_chip_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model treated age 18 as satisfying Wisconsin CHIP's modeled age test and therefore decided the case solely from its 299% FPL estimate. PolicyEngine's age criterion excludes dependent2 at age 18, so the asserted 300% income limit cannot produce eligibility." -us,scenario_064,dependent2_chip_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model applied a generic “under 19” rule and moved directly to Wisconsin's asserted 306% FPL threshold. The modeled CHIP age test fails for dependent2 at age 18, making the income comparison irrelevant." -us,scenario_064,dependent2_chip_eligible,gpt-5.5,llm_error,age_disability,False,"The model incorrectly classified age 18 as within the child CHIP age limit used by the engine. Dependent2 fails PolicyEngine's age criterion, regardless of whether household income falls within Wisconsin's CHIP limit." -us,scenario_064,dependent2_chip_eligible,gpt-5.6-sol,llm_error,age_disability,False,The model placed dependent2 in an upper CHIP income band after applying an incorrect under-19 age rule. PolicyEngine excludes dependent2 at age 18 under its CHIP age criterion; the person is also Medicaid-ineligible with category NONE. -us,scenario_064,dependent2_chip_eligible,grok-build-0.1,llm_error,age_disability,False,"The model incorrectly treated age 18 as passing the modeled CHIP age test, then relied on its 273% FPL estimate and an employer-coverage affordability exception. Dependent2 fails PolicyEngine's age criterion, so neither the income-band calculation nor ESI affordability changes the result." -us,scenario_064,dependent2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no output for dependent2_chip_eligible. The required parsed value was 0 because dependent2 is age 18.0 and fails PolicyEngine's CHIP age criterion. -us,scenario_064,dependent2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model substituted a 301% FPL children's coverage threshold for the applicable Wisconsin Medicaid category rules and then inferred eligibility from dependent2 having no personal income. Medicaid MAGI uses the relevant tax-unit household income, which is 3.22 times FPL here, and the engine assigns no qualifying category." -us,scenario_064,dependent2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model applied an inapplicable 306% FPL child threshold, added a 5% disregard as though it raised that threshold, and used its own family-of-five FPL arithmetic. The resulting 3.22-times-FPL MAGI exceeds the applicable Medicaid limits, leaving dependent2 with no eligibility category." -us,scenario_064,dependent2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model omitted the required dependent2_medicaid_eligible output. The required value is 0 because dependent2's Medicaid category is NONE at 3.22 times FPL and no alternative categorical pathway applies. -us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model derived taxable income close to the traced amount but applied approximate 2026 bracket thresholds and arithmetic, producing about $7,630 instead of $7,639.29. It then reported $5,959 despite its own stated $3,200 credit subtraction, which would place its result near $4,430." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $13,899 overtime premium by adding it to wages even though gross wages already include overtime, and it used a $14,600 single-filer-sized deduction instead of the $32,200 MFJ standard deduction. It also failed to apply the traced overtime and QBI deductions and did not calculate the $3,200 nonrefundable credits coherently." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented self-employment tax and its half-tax deduction despite the farm loss offset in the traced computation, and it omitted the $13,899.29 overtime deduction. It also split the child tax credit into an unsupported refundable portion instead of subtracting the full $3,200 of usable nonrefundable credits." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model mixed self-employment tax into an output expressly limited to federal income tax after nonrefundable credits, omitted the $13,899.29 overtime deduction, and then reported $14,959 even though its own estimated tax and credit arithmetic led to a much lower amount. The required computation is $7,639.29 minus $3,200, not an income-tax-plus-payroll-tax total." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated the domestic production input as an above-the-line deduction, overstated QBI, and omitted the $13,899.29 overtime deduction. Its reported $11,200 also conflicts with its own estimated $8,700 pre-credit tax and $2,500 credit subtraction." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented a $1,560 self-employed health-insurance deduction and a $989 half-SE-tax deduction, used a $30,000 rather than $32,200 standard deduction, and entirely omitted the $13,899.29 overtime deduction. These errors replaced the traced $67,794.08 taxable income with $81,873." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model omitted the $13,899.29 overtime deduction and then asserted that the child credit was mostly refundable despite sufficient tax liability for all $3,200 of nonrefundable credits. Its $12,500 result is also incompatible with its own estimate of roughly $9,300 before credits." -us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied pre-TCJA personal exemptions, a $15,900 standard deduction, 10% and 15% brackets, and a $1,000 child credit instead of the operative 2026 rules. It also deducted health premiums from wages and omitted the $13,899.29 overtime deduction and full $3,200 nonrefundable credits." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly removed employer-sponsored premiums from wages and treated those premiums and other listed medical costs as a $21,880 itemized deduction. It also used personal exemptions, mislabeled $1,796 as QBI instead of using the traced $510 deduction, and allowed only a $1,000 child credit rather than $3,200." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $11,090.45 result does not reflect the traced reduction from $114,403.37 AGI to $67,794.08 taxable income through the standard, QBI, and overtime deductions. It also fails to reflect the final $3,200 nonrefundable-credit subtraction from $7,639.29." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used sunset-era personal exemptions, a smaller standard deduction, a $1,000 child credit, and a 10% early-distribution penalty that is outside this income-tax output. It also deducted ESI and DPAD from AGI while omitting the $13,899.29 overtime deduction and $510 QBI deduction." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used personal exemptions and only a $1,000 child credit under an obsolete rule set. The operative computation uses the $32,200 standard deduction plus the overtime and QBI deductions, followed by $3,200 of nonrefundable credits." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model gave no derivation for $6,215, which exceeds the traced $4,439.29 by $1,775.71. Its answer fails to implement the exact $67,794.08 taxable-income calculation and $3,200 nonrefundable-credit subtraction." -us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response could not be parsed or evaluated as a substantive calculation." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated medical and retirement items plus dependent credits as sufficient to eliminate liability. The allowed deductions leave $67,794.08 taxable income and $7,639.29 of pre-credit tax, while the $3,200 nonrefundable credits leave $4,439.29 rather than zero." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed unspecified large deductions and credits eliminated the tax. The traced deductions produce $67,794.08 of taxable income, and the available $3,200 nonrefundable credits do not eliminate the resulting $7,639.29 tax." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model followed the correct structure but used AGI of $114,422 and taxable income of $67,813 instead of the traced $114,403.37 and $67,794.08. That parameter and rounding drift produced $7,641.56 before credits rather than $7,639.29." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The $2,219 answer is exactly one $2,220-sized credit below the traced neighborhood and reflects an extra child-credit subtraction or refundable-credit treatment. Only $3,200 of nonrefundable credits reduces the $7,639.29 pre-credit tax in this output." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model named the relevant deductions and credits but its $4,485 estimate used approximate parameters or bracket arithmetic. Exact taxable income is $67,794.08, exact regular tax is $7,639.29, and subtracting $3,200 yields $4,439.29." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model referred only to a qualifying-child credit and omitted the two $500 credits for the disabled 27-year-old and the 18-year-old dependents. The computation subtracts $3,200 in total nonrefundable child and other-dependent credits." -us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model relied on gross-income intuition and did not apply the traced $46,609.29 of below-AGI deductions, including the $13,899.29 overtime deduction. It also failed to subtract the full $3,200 of nonrefundable dependent credits." -us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied TCJA-sunset personal exemptions, a $17,000 standard deduction, 10% and 15% brackets, and a phased $1,000 child credit. The operative 2026 calculation instead uses a $32,200 standard deduction, the overtime and QBI deductions, current brackets, and $3,200 of unphased nonrefundable credits." -us,scenario_064,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model deducted $21,208 of ESI and health premiums from wages, double-counted the $14,000 self-employment partnership amount, and omitted the $13,899.29 overtime deduction. It also substituted a $1,796 QBI figure for the traced $510 and allowed only the qualifying-child credit rather than all $3,200 of credits." -us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model double-counted $14,000 of self-employment partnership income on top of the reported $19,350 partnership/S-corp income and deducted retirement contributions belonging to a dependent's separate income calculation. It also overstated QBI to $6,670, omitted the $13,899.29 overtime deduction, and omitted one $500 other-dependent credit." -us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model failed to reduce wages by the $2,701.30 pre-tax 401(k) contribution and substituted the $1,796 domestic production input for the traced $510 QBI deduction. Those errors raised taxable income from $67,794.08 to $68,842 and therefore raised tax after the correctly identified $3,200 credits." -us,scenario_064,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer implies that deductions and nonrefundable credits were treated as eliminating all liability. The traced deductions leave $67,794.08 taxable income, and $3,200 of credits reduces $7,639.29 only to $4,439.29." -us,scenario_064,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model deducted the entire $18,235 capital loss rather than applying the traced treatment, omitted the taxable-income reductions for overtime and QBI, and wrongly denied nonrefundable credits at an income far below the phaseout threshold. Its reported $14,629.40 also contradicts its own stated $7,629.40 pre-credit tax." -us,scenario_064,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model counted the full $2,000 Child Tax Credit for the 12-year-old as refundable federal credits. That credit is nonrefundable to the extent it is absorbed by federal income tax liability, and the household has no refundable additional CTC after the 2026 refundable-credit computation." -us,scenario_064,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model identified that the CTC is absorbed nonrefundably by substantial tax liability, then still submitted the $1,700 ACTC cap as if the refundable portion applied. The ACTC cap is only a ceiling on a refundable amount after the statutory computation, and this household's refundable CTC amount is $0." -us,scenario_064,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model did not provide a parseable federal_refundable_credits value or supporting reasoning. This is a contract failure rather than a substantive tax calculation. -us,scenario_064,federal_refundable_credits,kimi-k2.6,llm_error,credit_phaseout,False,"The submitted $2,351 is consistent with adding a positive refundable credit despite the reference computation finding no EITC, refundable CTC, refundable AOTC, Recovery Rebate Credit, or refundable payroll tax credit. The model failed to separate nonrefundable child/dependent credits and regular income tax liability from the refundable-credit-only output." -us,scenario_064,free_school_meals_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the farm loss and capital loss from school-meal countable income. The applicable calculation instead counts $97,295 of employment income and $20,000 of retirement distributions, producing $117,295 and the paid tier; no SNAP/TANF categorical pathway overrides that result." +us,scenario_064,dependent1_medicaid_eligible,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model attributed the head and spouse's tax-unit MAGI to dependent1. PolicyEngine instead computes dependent1's relevant MAGI income level as 0.00 times FPL, which satisfies the ACA adult expansion threshold." +us,scenario_064,dependent1_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied an asset test using assets owned by the household and treated disability-based Medicaid as the controlling pathway. PolicyEngine qualifies dependent1 through the MAGI-based ACA adult expansion category, which uses the traced 0.00-times-FPL income level rather than those household assets." +us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model substituted the household's combined income for dependent1's Medicaid MAGI. The traced person-level MAGI is 0.00 times FPL, placing this 27-year-old within the ACA adult expansion limit." +us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,The model assigned the head's substantial income to dependent1 and concluded that both MAGI and disability pathways failed. The controlling adult expansion pathway uses dependent1's traced MAGI level of 0.00 times FPL and returns eligibility. +us,scenario_064,dependent1_medicaid_eligible,claude-opus-5,llm_error,household_unit_or_filing_status,False,The model treated the adult dependent's Medicaid income as the entire household's income. PolicyEngine calculates dependent1 at 0.00 times FPL and qualifies them through the ACA adult expansion category. +us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model aggregated the head's wages, distributions, partnership income, and other amounts into dependent1's Medicaid MAGI. The engine's applicable person-level MAGI measure is 0.00 times FPL, below the adult expansion threshold." +us,scenario_064,dependent1_medicaid_eligible,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model used household income to deny adult Medicaid and also invoked Wisconsin's 100% FPL limit instead of the benchmark's applicable expansion threshold. Dependent1's traced MAGI is 0.00 times FPL, so the ACA adult category is satisfied." +us,scenario_064,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,The model treated household income as dependent1's applicable Medicaid income. PolicyEngine derives a 0.00-times-FPL MAGI level for dependent1 and qualifies them as an expansion adult. +us,scenario_064,dependent1_medicaid_eligible,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model compared the broader household's income with Wisconsin's adult threshold. The relevant engine computation assigns dependent1 a MAGI income level of 0.00 times FPL, which passes the ACA adult expansion test." +us,scenario_064,dependent1_medicaid_eligible,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model included the tax unit's roughly $100,000-plus income in dependent1's MAGI and also treated employer-sponsored insurance as disqualifying. PolicyEngine computes dependent1 at 0.00 times FPL, and existing employer coverage does not defeat eligibility under the adult expansion category." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,The answer's high-income-household rationale substitutes household income for dependent1's relevant Medicaid MAGI. The engine computes dependent1 at 0.00 times FPL and returns eligibility through adult expansion. +us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model required additional disability-related details and used a simplified household-income assumption. Disability documentation is unnecessary because dependent1 qualifies through the under-65 ACA adult category at a traced MAGI level of 0.00 times FPL. +us,scenario_064,dependent1_medicaid_eligible,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model assigned the family's annual income to dependent1 merely because they are a tax dependent. PolicyEngine's applicable computation gives dependent1 a MAGI income level of 0.00 times FPL, below the adult expansion limit." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-luna,llm_error,asset_resource,False,"The model invoked income and resource rules without applying the successful MAGI adult pathway. Dependent1's traced MAGI is 0.00 times FPL, and an asset test does not override eligibility in that category." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-terra,llm_error,asset_resource,False,"The model treated the household's income and assets as defeating Medicaid eligibility. PolicyEngine qualifies dependent1 under the MAGI-based ACA adult category at 0.00 times FPL, where those household assets are not the controlling test." +us,scenario_064,dependent1_medicaid_eligible,grok-4.3,llm_error,household_unit_or_filing_status,False,The model's threshold conclusion implies that it used the household's income rather than dependent1's engine-calculated MAGI. Dependent1 is at 0.00 times FPL and therefore passes the adult expansion threshold. +us,scenario_064,dependent1_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"The model made the non-MAGI aged, blind, or disabled resource test dispositive and also declared the MAGI test failed. PolicyEngine instead returns eligibility through the ACA adult expansion category at 0.00 times FPL, for which the cited asset test does not apply." +us,scenario_064,dependent1_medicaid_eligible,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model calculated dependent1 at roughly 273% FPL by assigning household income to them, then added an ABD asset test. The engine calculates dependent1 at 0.00 times FPL and qualifies them through MAGI adult expansion, so neither shortcut controls." +us,scenario_064,dependent1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for the requested variable, so it failed the required output contract." +us,scenario_064,dependent1_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,The unexplained denial omits the successful ACA adult expansion pathway. PolicyEngine classifies the 27-year-old as an adult with MAGI at 0.00 times FPL and therefore returns eligible. +us,scenario_064,dependent1_medicaid_eligible,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,The model included the head's wages and other income in dependent1's Medicaid MAGI and used an approximately 100% FPL adult limit. PolicyEngine computes dependent1 at 0.00 times FPL and applies the ACA adult expansion category. +us,scenario_064,dependent1_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model stated that dependent1's income was too high even though no income was listed for that person. The engine derives dependent1's MAGI income level as 0.00 times FPL, which satisfies the adult expansion test." +us,scenario_064,dependent1_medicare_eligible,claude-fable-5,llm_error,age_disability,False,The model converted the listed disability flag directly into SSDI-based Medicare eligibility. It failed to require SSDI entitlement for the applicable waiting period or another qualifying condition such as ALS or end-stage renal disease. +us,scenario_064,dependent1_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model incorrectly treated disability as sufficient for Medicare eligibility regardless of age. A 27-year-old needs a distinct under-65 Medicare pathway, and no SSDI entitlement, qualifying duration, ALS, or end-stage renal disease was listed." +us,scenario_064,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model stated the SSDI entitlement and 24-month requirements but then replaced them with the bare disability flag. Constant disability status does not establish receipt of SSDI or completion of Medicare's waiting period, and unlisted SSDI, ALS, and end-stage renal disease statuses are false." +us,scenario_064,dependent1_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated reported disability as an automatic under-65 Medicare qualification. It omitted the required SSDI-entitlement pathway and its duration requirement, as well as the alternative ALS or end-stage renal disease pathways." +us,scenario_064,dependent2_chip_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model treated age 18 as satisfying Wisconsin CHIP’s age test and awarded eligibility from an asserted 300% FPL limit. Under the modeled rule, dependent2 fails the CHIP age criterion at 18, so the 299% FPL calculation cannot make the person eligible." +us,scenario_064,dependent2_chip_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model applied a generic under-19 age rule and proceeded directly to a 306% FPL income threshold. PolicyEngine’s Wisconsin CHIP age test excludes this 18-year-old, who is also not Medicaid-eligible." +us,scenario_064,dependent2_chip_eligible,gpt-5.5,llm_error,age_disability,False,"The model explicitly classified age 18 as within the child CHIP age limit. The modeled Wisconsin rule excludes dependent2 at age 18, so being within an asserted child income limit does not establish eligibility." +us,scenario_064,dependent2_chip_eligible,gpt-5.6-sol,llm_error,age_disability,False,The model applied an under-19 age rule and then placed dependent2 in an upper CHIP income band above Medicaid. The engine instead excludes the person under CHIP’s age criterion at age 18 and returns no Medicaid eligibility. +us,scenario_064,dependent2_chip_eligible,grok-build-0.1,llm_error,age_disability,False,"The model treated age 18 as satisfying CHIP’s under-19 rule and relied on its 273% FPL estimate plus an ESI-affordability argument. The modeled Wisconsin CHIP age test excludes dependent2 at 18, so neither the income estimate nor the treatment of employer coverage changes the result." +us,scenario_064,dependent2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for dependent2_chip_eligible. It therefore failed the required output contract instead of returning the engine-derived value 0. +us,scenario_064,dependent2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied a 301% FPL children's Medicaid threshold and asserted that family size brought dependent2 under it. The engine computes MAGI at 3.22 times FPL, above the applicable limits, so dependent2 receives no Medicaid category." +us,scenario_064,dependent2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model used an incorrect Wisconsin Medicaid cutoff of 306% FPL, added a 5% disregard to that cutoff, and compared household income with an independently estimated family-of-five poverty amount. The engine's MAGI calculation is 3.22 times FPL and yields no qualifying Medicaid category for dependent2." +us,scenario_064,dependent2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model omitted the required output entirely. The eligibility derivation yields 0 because dependent2 is at 3.22 times FPL and qualifies through none of Wisconsin's Medicaid pathways. +us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model derived taxable income close to the trace but used estimated bracket parameters, producing about $7,630 rather than $7,639.29, and its stated subtraction does not arithmetically yield $5,959. Applying the exact 2026 brackets and the full $3,200 nonrefundable credits yields $4,439.29." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $13,899 overtime premium by adding it to wages even though gross wages already include overtime, and then used a $14,600 single-filer-sized deduction instead of the $32,200 MFJ standard deduction. It also omitted the $13,899.29 overtime deduction and did not correctly apply the $3,200 dependent credits." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented self-employment tax and its half-tax deduction from the separately listed $14,000 input, then omitted the $13,899.29 overtime deduction. It also treated part of the available child tax credit as refundable despite sufficient regular tax to use all $3,200 of nonrefundable credits." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model mixed self-employment tax into the requested individual-income-tax output and then reported $14,959 despite its own intermediate estimate of roughly $9,200 before credits. The requested value excludes self-employment tax and equals $7,639.29 less $3,200 of nonrefundable credits." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated the domestic production input as an AGI deduction, overstated QBI, and omitted the $13,899.29 overtime deduction. Those errors displaced the traced $67,794.08 taxable income and its $4,439.29 post-credit tax." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented a $1,560 self-employed health-insurance deduction and a $989 half-SE-tax deduction, used a $30,000 standard deduction, and omitted the $13,899.29 overtime deduction. It also reduced QBI to nearly zero instead of applying the traced $510 deduction." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $13,899.29 overtime deduction, treated the domestic production input as an above-the-line deduction, and understated the usable nonrefundable child credit. Its later addition of unspecified Medicare or NIIT amounts has no basis because neither tax belongs in this output under these facts." +us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied assumed TCJA-sunset rules with personal exemptions, a $15,900 standard deduction, 15% brackets, and a $1,000 child credit. The applicable 2026 computation instead uses the $32,200 MFJ standard deduction, the overtime and QBI deductions, current brackets, and $3,200 of nonrefundable dependent credits." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted employer-sponsored premiums from wages, fabricated a $21,880 itemized medical deduction, and applied personal exemptions. The trace uses $94,979.60 of employment income after only the pre-tax 401(k), then the $32,200 standard, $13,899.29 overtime, and $510 QBI deductions." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $11,090.45 does not reflect the traced reduction from $114,403.37 AGI to $67,794.08 taxable income or the subsequent $3,200 nonrefundable credits. Applying those steps yields $4,439.29." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used obsolete TCJA-sunset personal exemptions and a reduced standard deduction, while omitting the overtime deduction. It also added a $2,000 early-distribution penalty to an output that measures regular federal income tax after nonrefundable credits, not additional retirement-distribution taxes." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied personal exemptions and only a $1,000 child credit under an obsolete sunset-law framework. The traced computation has no personal exemptions and applies $3,200 of nonrefundable child and other-dependent credits after calculating $7,639.29 of regular tax." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer fails to incorporate the complete traced deduction stack: $32,200 standard deduction, $13,899.29 overtime deduction, and $510 QBI deduction. These produce $67,794.08 of taxable income and $4,439.29 after the $3,200 credits." +us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated medical and retirement amounts as large enough to erase liability, but the trace leaves $67,794.08 of taxable income after all allowed deductions. The $3,200 nonrefundable credits reduce $7,639.29 of tax only to $4,439.29, not zero." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed deductions and credits eliminate the tax without computing their statutory limits. Allowed deductions leave $67,794.08 taxable, and the available $3,200 credits do not exhaust the resulting $7,639.29 tax." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model followed the correct structure but used $114,422 AGI and $67,813 taxable income instead of the traced $114,403.37 and $67,794.08, then calculated regular tax $2.27 too high. Exact income adjustments and 2026 brackets give $7,639.29 before the same $3,200 credits." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The $2,219 result is consistent with subtracting excessive or duplicated child-related credits from regular tax. Only $3,200 of nonrefundable child and other-dependent credits reduce the traced $7,639.29 tax, leaving $4,439.29." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model named the relevant income, deduction, and credit categories but its rounded estimate did not use their exact traced amounts. Exact taxable income of $67,794.08 produces $7,639.29 of regular tax and $4,439.29 after credits." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The result understates the nonrefundable dependent credits by $1,015.71 relative to the traced computation. One qualifying child and two other dependents generate $3,200 of usable credits, not merely a qualifying-child credit." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model relied on gross household income and did not apply the traced $46,609.29 deduction stack or the $3,200 nonrefundable credits. Those reductions bring the requested liability to $4,439.29." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied obsolete TCJA-sunset personal exemptions, a $17,000 standard deduction, 15% brackets, and a phased $1,000 child credit. The applicable 2026 rules use the $32,200 MFJ standard deduction, the overtime and QBI deductions, and $3,200 of unphased nonrefundable credits." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted employer-sponsored insurance and health premiums from wages, double-counted the separately reported $14,000 self-employment subset as additional income, and omitted the $13,899.29 overtime deduction. It also used the $1,796 domestic-production input in place of the traced $510 QBI deduction and allowed only $2,000 rather than $3,200 of credits." +us,scenario_064,federal_income_tax_before_refundable_credits,inkling,llm_error,other,False,"The model incorrectly added a $2,000 early-withdrawal penalty to the requested regular-income-tax output and substituted a $1,796 domestic-production deduction for the traced overtime and QBI deductions. The taxable distribution belongs in income, but its additional early-distribution tax is outside this output." +us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model double-counted the $14,000 self-employment partnership amount on top of the $19,350 partnership/S-corp income and deducted retirement contributions belonging to a dependent's separate income calculation. It also omitted the $13,899.29 overtime deduction, overstated QBI at $6,670 instead of $510, and omitted one $500 other-dependent credit." +us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model failed to reduce employment income by the $2,701.30 pre-tax 401(k) contributions and substituted the $1,796 domestic-production input for the traced $510 QBI deduction. Those differences make its taxable income $1,047.92 too high; the traced taxable income is $67,794.08." +us,scenario_064,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer assumes deductions and credits erase liability, but allowed deductions leave $67,794.08 taxable income. The resulting $7,639.29 tax exceeds the $3,200 nonrefundable credits by $4,439.29." +us,scenario_064,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model directly included the full long-term capital loss rather than following the traced income and loss-deduction treatment, omitted the overtime and QBI deductions, and then declared all dependent credits unavailable despite AGI being well below their MFJ phaseout. It also reported $14,629.40 even though its own stated pre-credit tax was $7,629.40." +us,scenario_064,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model simultaneously subtracted a standard deduction and fabricated itemized medical and retirement deductions, invented a $12,150 child credit, and then produced a final liability larger than its own pre-credit tax. The valid computation subtracts $3,200 of credits from $7,639.29, so credits reduce rather than increase the result." +us,scenario_064,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated the full $2,000 Child Tax Credit for the age-12 child as refundable. The household's tax liability absorbs that credit nonrefundably, leaving no Additional Child Tax Credit and total federal refundable credits of $0." +us,scenario_064,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly stated that the household's substantial tax liability absorbs the CTC nonrefundably, then incorrectly submitted the $1,700 ACTC per-child cap as though the cap were an automatic refundable amount. A cap limits an otherwise computed refundable portion; it does not create one when the CTC is fully used against tax liability." +us,scenario_064,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value for federal_refundable_credits, violating the required outputs contract." +us,scenario_064,federal_refundable_credits,kimi-k2.6,llm_error,credit_phaseout,False,"The unexplained $2,351 answer assigns a positive refundable amount even though EITC and every other refundable component are zero and the child credit is fully absorbed nonrefundably. The correct refundable-credit aggregation sums those components to $0." +us,scenario_064,free_school_meals_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the negative farm income and capital loss when estimating school-meal countable income. The applicable calculation counts $97,295 of employment income plus $20,000 of retirement distributions, producing $117,295 and a 3.03 FPG ratio, so the household exceeds even the 185% reduced-price threshold and receives no free-meal support." us,scenario_064,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_064,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_064,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model correctly computed federal FICA of $7,443.07 but then added a fictitious Wisconsin employee-side payroll tax and an unexplained adjustment. Wisconsin contributes no mandatory employee-side state payroll tax to this output, so the total stops at Social Security plus Medicare on the head's wages." -us,scenario_064,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model's reasoning derives the correct payroll tax exactly: $6,032.29 of Social Security plus $1,410.78 of Medicare equals $7,443.07. Its submitted value of $7,407.36 contradicts its own calculation, so the error is a final-answer transcription or arithmetic failure rather than a rule error." -us,scenario_064,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model wrongly subtracted $1,560 of health insurance premiums from the head's FICA wage base. Employee Social Security and Medicare taxes apply to the full listed $97,295 of gross wages in this benchmark, not to gross wages net of health premiums." -us,scenario_064,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model reduced the FICA wage base by employer-sponsored insurance premiums and desired traditional 401(k) contributions, treating income-tax or benefit inputs as payroll-tax exclusions. PolicyEngine's payroll_tax calculation uses the head's full $97,295 of gross wages for employee Social Security and Medicare tax." -us,scenario_064,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted the $19,648 employer-sponsored insurance premium from wages before applying the 7.65% FICA rate. The payroll_tax output uses gross wages of $97,295 as the employee FICA base, so the premium does not reduce Social Security or Medicare wages here." -us,scenario_064,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model treated employer-sponsored insurance premiums as a pre-tax reduction to FICA taxable wages and applied FICA to $77,647 instead of $97,295. PolicyEngine calculates employee Social Security and Medicare tax on the full wage amount supplied for the head." -us,scenario_064,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model did not provide a parseable payroll_tax value or explanation. This is a missing-output failure, not a substantive payroll-tax calculation." -us,scenario_064,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The submitted $12,306 is far above 7.65% of the head's $97,295 wages and is consistent with adding non-wage income or otherwise expanding the FICA base beyond wages. The payroll_tax output includes employee Social Security and Medicare on wages only, while self-employment or partnership income belongs outside this employee-side payroll-tax value." -us,scenario_064,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The submitted $8,659 exceeds the 7.65% employee FICA tax on the head's $97,295 wages and reflects an inflated wage base or an extra payroll component. The correct computation applies only Social Security and Medicare rates to the head's listed gross wages, with no Additional Medicare Tax at this income level and no Wisconsin employee payroll tax." -us,scenario_064,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable payroll_tax value or explanation. This is a missing-output failure, not a substantive payroll-tax calculation." -us,scenario_064,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model ignored the head's $97,295 of wage income and returned zero payroll tax. Those wages are subject to employee Social Security and Medicare taxes, producing a positive employee-side payroll_tax." -us,scenario_064,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly combined partnership/S-corporation income, self-employment partnership income, and the farm loss into an invented positive self-employment-tax base. It also wrongly treated the income-tax deduction for one-half of self-employment tax as a reduction in the self-employment tax liability itself; the computed net earnings subject to the tax are zero." -us,scenario_064,self_employment_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model discarded the offsetting farm loss and taxed the $14,000 self-employment partnership input in isolation. The person-level computation produces no positive net earnings subject to self-employment tax, so applying the 92.35% adjustment and 15.3% rate to $14,000 was the wrong tax-base step." -us,scenario_064,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model correctly reached a nonpositive result when combining the stated self-employment amounts, then abandoned that result and invented a $15,900 positive base by mixing the broader $19,350 partnership/S-corporation income with the $14,000 self-employment partnership amount. The computation contains zero net earnings subject to self-employment tax, so no positive component is separately taxed." -us,scenario_064,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model treated the $14,000 self-employment partnership input as a standalone taxable base and applied the self-employment tax rate without determining person-level net earnings from all self-employment activities. That computation yields zero earnings subject to self-employment tax and therefore no liability." -us,scenario_064,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable self_employment_tax output. The required numeric value was $0 because every person's net earnings subject to self-employment tax are zero. -us,scenario_064,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model taxed a positive self-employment partnership amount after the 92.35% adjustment while omitting the computation of net earnings across the person's self-employment activities. The resulting taxable self-employment earnings are zero, so no 15.3% tax applies." -us,scenario_064,self_employment_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model mechanically applied the 92.35% adjustment and the Social Security and Medicare self-employment rates to $14,000 as though it were the final net-earnings base. The person-level tax base is zero, so neither component of self-employment tax is imposed." -us,scenario_064,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable self_employment_tax output. The required numeric value was $0 because every person's net earnings subject to self-employment tax are zero. -us,scenario_064,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model repeatedly derived nonpositive combined self-employment income, then contradicted that derivation by taxing the $14,000 partnership amount separately on the unsupported premise that the farm loss did not offset it. The computation produces zero net earnings subject to self-employment tax, so the positive partnership input cannot be isolated and taxed at 15.3%." +us,scenario_064,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model invented a Wisconsin employee payroll tax despite Wisconsin having no mandatory employee-side state payroll contribution in this calculation. It then replaced its own $10,361.92 sum with an unexplained $12,318.64 adjustment instead of returning the $7,443.07 federal FICA total." +us,scenario_064,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model derived the exact correct components—$6,032.29 of Social Security tax and $1,410.78 of Medicare tax—and explicitly summed them to $7,443.07, but submitted $7,407.36. This is a final-answer transcription or arithmetic inconsistency." +us,scenario_064,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the separately listed $1,560 health-insurance premium from Social Security and Medicare wages. FICA applies to the full $97,295 wage input here, producing $7,443.07." +us,scenario_064,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages by both the $19,648 employer-sponsored insurance premium and the $2,315 traditional 401(k) contribution. Neither deduction reduces the benchmark’s payroll-tax wage base, so Social Security and Medicare apply to the full $97,295." +us,scenario_064,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated the $19,648 employer-sponsored insurance premium as a pre-tax employee deduction and subtracted it from FICA wages. The benchmark supplies $97,295 as the applicable wage base, yielding $7,443.07 at the combined 7.65% FICA rate." +us,scenario_064,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $19,648 employer-sponsored insurance premium from gross wages before applying FICA. Social Security and Medicare apply to the full $97,295 wage amount in this case." +us,scenario_064,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, violating the required structured-output contract." +us,scenario_064,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The submitted $12,306 implies a FICA wage base of roughly $160,863, far above the only listed wages of $97,295. The model folded non-wage amounts into the employee payroll-tax base even though farm, partnership, and self-employment income do not enter this output." +us,scenario_064,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The $8,659 estimate implies about $113,190 of FICA wages, exceeding the listed $97,295 wage income. The model added non-wage income or another unsupported amount to the employee payroll-tax base; Additional Medicare Tax is zero at this income." +us,scenario_064,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, violating the required structured-output contract." +us,scenario_064,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer ignored the head’s $97,295 of wages. Those wages generate $6,032.29 of employee Social Security tax and $1,410.78 of employee Medicare tax." +us,scenario_064,payroll_tax,qwen3.8-max,llm_error,other,False,"The model correctly calculated $6,032.29 of Social Security tax and $1,410.78 of Medicare tax with no Additional Medicare Tax, but submitted $8,529.65 instead of their $7,443.07 sum. Its final value includes an unsupported extra $1,086.58." +us,scenario_064,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model constructed a self-employment tax base by combining the $14,000 self-employment partnership input, $19,350 partnership/S-corp income, and the farm loss, even though the computed person-level taxable net earnings are zero. It also incorrectly treated the deduction for one-half of self-employment tax as a reduction of the tax liability itself; that deduction affects adjusted gross income, not self-employment tax." +us,scenario_064,self_employment_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model directly treated the $14,000 self-employment partnership input as taxable net earnings and applied the 92.35% adjustment and 15.3% rate. PolicyEngine’s computed self-employment-tax base is zero for the head and every other person, so there is no amount to which those factors apply." +us,scenario_064,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model correctly reached a nonpositive result when netting the $14,000 against the farm loss, then abandoned that result and invented a $15,900 positive base by mixing the broader $19,350 partnership/S-corp amount with its $14,000 self-employment subset. The computed person-level net-earnings base is zero, so neither the positive-only treatment nor the double-counted reconstruction is valid." +us,scenario_064,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model applied the self-employment tax formula directly to the $14,000 self-employment partnership input. It skipped the required computation of the taxable person-level net-earnings base, which is zero for all household members." +us,scenario_064,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable self_employment_tax output. The required numeric output was $0 because every person’s computed net earnings subject to self-employment tax are zero. +us,scenario_064,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model asserted a positive self-employment partnership tax base without deriving it from the person-level net-earnings computation. That computation yields zero taxable self-employment earnings for every person, so no standard net-earnings adjustment or self-employment tax rate applies." +us,scenario_064,self_employment_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model mechanically multiplied the $14,000 input by 92.35% and then by the Social Security and Medicare self-employment rates. It mistook that input for the computed taxable net-earnings base, which is zero across all five people." +us,scenario_064,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable self_employment_tax output. The required numeric output was $0 because the computation produces no net earnings subject to self-employment tax for any household member. +us,scenario_064,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model repeatedly derived a zero or negative combined self-employment base after accounting for the farm loss, but discarded that derivation and taxed the $14,000 positive component in isolation. Self-employment tax is based on the computed net earnings rather than grossing up a selected positive component, and that computed base is zero for every person." us,scenario_064,snap,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_064,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_064,ssi,deepseek-v4-pro,llm_error,age_disability,False,"The model treated Dependent 1's generic disability status as proof of SSI categorical eligibility, but Dependent 1 has `is_ssi_aged_blind_disabled=False`. It then incorrectly applied an in-kind-support reduction to a benefit calculation that never begins because no member satisfies SSI's aged, blind, or disabled criterion." -us,scenario_064,ssi,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model converted the generic `is disabled` fact directly into a qualifying SSI disability determination. Dependent 1's SSI-specific aged/blind/disabled predicate is false, so the maximum federal SSI rate does not apply." -us,scenario_064,ssi,gemini-3.5-flash,llm_error,age_disability,False,"The model assumed that Dependent 1's generic disability status establishes SSI categorical eligibility and proceeded to combine a federal benefit with a Wisconsin supplement. Dependent 1 has `is_ssi_aged_blind_disabled=False`, so neither the federal SSI amount nor a state supplement is payable." -us,scenario_064,ssi,glm-5.2,llm_error,age_disability,False,"The model treated the 27-year-old's generic disability status as a qualifying adult SSI disability determination, then analyzed deeming, income, resources, and projected federal benefit rates. Because Dependent 1's SSI-specific aged/blind/disabled predicate is false, those downstream calculations are irrelevant and SSI is zero." -us,scenario_064,ssi,gpt-5.5,llm_error,age_disability,False,"The model assumed the disabled adult dependent had already met SSI's categorical disability requirement and awarded the maximum federal rate based on zero personal income and resources. Dependent 1 has `is_ssi_aged_blind_disabled=False`, so no SSI benefit is calculated." -us,scenario_064,ssi,gpt-5.6-sol,llm_error,age_disability,False,"The model equated Dependent 1's generic disability fact with SSI categorical eligibility and multiplied the maximum monthly federal rate by 12. The SSI-specific predicate is false for Dependent 1 and every other household member, making the annual amount zero." -us,scenario_064,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable SSI value or explanation, violating the required output contract." -us,scenario_064,ssi,kimi-k3,llm_error,age_disability,False,"The model declared Dependent 1 categorically SSI-eligible solely from the generic disability fact, then applied the full federal benefit rate after correctly noting that parental deeming ends at age 18. Dependent 1's SSI-specific aged/blind/disabled predicate is false, so deeming, resources, and the benefit rate never enter the calculation." -us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model replaced the traced $114,403.37 AGI and $8,759.12 Wisconsin standard deduction with its own capital-loss adjustment and an approximately $11,300 deduction. It then subtracted unsupported nonrefundable credits to force tax from its own $4,964 bracket calculation down to $2,699." -us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model constructed an approximate $104,434 income base by directly combining gross income, the full capital loss, deductions, and exemptions instead of using Wisconsin AGI of $114,403.37 and taxable income of $102,144.25. Applying the 2026 Wisconsin schedule to the traced taxable income yields $4,606, not $4,512." -us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used an estimated standard deduction, counted only four exemptions instead of five, and then subtracted approximately $875 of school-property and itemized-deduction credits without inputs establishing those credits. The traced calculation uses a $8,759.12 deduction and $3,500 of exemptions, producing $102,144.25 of taxable income and $4,606." -us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model estimated Wisconsin AGI near $113,000 and taxable income near $100,000 rather than carrying through $114,403.37 of AGI, the $8,759.12 standard deduction, and $3,500 of exemptions. It also described personal exemptions inconsistently as a credit-equivalent adjustment, producing a result above the scheduled tax on $102,144.25." -us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model rounded the bracket result to $4,600 instead of completing the Wisconsin calculation on taxable income of $102,144.25. The 2026 joint rate schedule produces exactly $4,606 before refundable credits." -us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model overstated Wisconsin AGI at $116,040 and understated the sliding standard deduction at $3,957, creating taxable income of $108,583 instead of $102,144.25. It then invented approximately $520 of nonrefundable credits despite identifying no supported credit, rather than using the traced $8,759.12 deduction and five exemptions." -us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $14,000 self-employment partnership amount alongside the $19,350 partnership/S-corporation income and treated the domestic production deduction as an AGI subtraction. It also used only four exemptions and an estimated standard deduction, whereas the trace uses AGI of $114,403.37, a $8,759.12 deduction, and five exemptions." -us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the $1,796 domestic production deduction from AGI and applied a flat $25,000 Wisconsin standard deduction. Wisconsin's income-dependent deduction is $8,759.12 here, leaving $102,144.25 taxable rather than $82,566." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used a $13,506 standard deduction instead of $8,759.12, reducing taxable income to $97,576 rather than $102,144.25. It also subtracted an unsupported $150 dependent credit, while the requested pre-refundable liability is $4,606 under the traced calculation." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied only a generic estimate and did not compute from the traced Wisconsin inputs. Applying the $8,759.12 standard deduction and $3,500 exemption subtraction to $114,403.37, then the 2026 joint schedule, yields $4,606 rather than $4,652.12." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated Wisconsin AGI as $92,978 instead of $114,403.37 and consequently used taxable income near $78,969 rather than $102,144.25. That incorrect income base caused the $934 understatement." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced the household to $82,256 of taxable income, omitting or overstating deductions relative to the traced $102,144.25. The correct base follows from $114,403.37 less $8,759.12 and $3,500." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model gave an unsupported rounded estimate without deriving Wisconsin taxable income or applying the brackets. The traced deductions leave $102,144.25 taxable, whose 2026 joint-schedule liability is $4,606." -us,scenario_064,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the required structured result was missing." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model asserted that deductions and credits eliminate the tax, but the specified adjustments leave $102,144.25 of Wisconsin taxable income. No supported nonrefundable credits erase the $4,606 liability generated by the 2026 joint schedule." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model assumed unspecified deductions and credits exceeded Wisconsin tax. The traced $8,759.12 standard deduction and $3,500 of exemptions leave $102,144.25 taxable and $4,606 due before refundable credits." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,166 standard deduction instead of $8,759.12 and counted only two $700 exemptions instead of all five. It then subtracted an unsupported $472 itemized or medical-expense credit, compounding the understatement." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The model supplied a rounded estimate without computing the Wisconsin income base, deduction, exemptions, or bracket tax. The traced computation leaves $102,144.25 taxable and produces $4,606." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model identified the relevant deduction and joint-rate framework but overstated the resulting tax by $266. Using the traced taxable income of $102,144.25 with the actual 2026 Wisconsin joint brackets yields $4,606, not $4,872." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"The model gave an unsupported estimate and did not establish the taxable-income or bracket computation. Wisconsin AGI of $114,403.37 less the $8,759.12 deduction and $3,500 exemptions yields $102,144.25 and tax of $4,606." -us,scenario_064,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model rounded the liability to $4,500 without calculating the traced deductions and brackets. Tax on the resulting $102,144.25 under the 2026 joint schedule is $4,606." -us,scenario_064,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated the sliding standard deduction at approximately $6,500 instead of $8,759.12 and therefore overstated taxable income at roughly $107,900. The trace yields $102,144.25 after five exemptions, producing $4,606 rather than $4,900." -us,scenario_064,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used 2024 Wisconsin brackets and obsolete 4.75% and 5.75% rates rather than the 2026 joint schedule. It also started from AGI of $106,225 and an estimated $13,500 deduction instead of the traced $114,403.37 AGI and $8,759.12 deduction." -us,scenario_064,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the required structured result was missing." -us,scenario_064,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model applied a nonexistent 6.27% marginal rate to income in this household's bracket instead of Wisconsin's 5.3% rate. It also used an understated $5,481 standard deduction, whereas the trace uses $8,759.12 and produces $102,144.25 of taxable income." -us,scenario_064,state_income_tax_before_refundable_credits,minimax-m3,llm_error,missing_output,False,"The model submitted zero without any substantive explanation or computation. The specified Wisconsin deductions leave $102,144.25 taxable, so the 2026 joint schedule produces $4,606 rather than zero." -us,scenario_064,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model wrongly excluded the $13,899 FLSA overtime premium from Wisconsin income and reduced wages to $83,396, even though gross wages already include all overtime and the trace's AGI is $114,403.37. It then used an estimated $13,500 standard deduction and incorrect bracket structure instead of the $8,759.12 deduction and 2026 Wisconsin joint schedule." -us,scenario_064,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model correctly ruled out Wisconsin EITC, homestead, farmland preservation, and veterans property-tax credits, but then submitted a $154 refundable credit unsupported by any Wisconsin refundable-credit pathway in its own reasoning. Its number is consistent with carrying forward a stray or imagined residual state refundable credit after concluding that every applicable Wisconsin refundable credit equals zero." -us,scenario_064,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model did not provide a parseable value for state_refundable_credits. The required Wisconsin refundable-credit computation yields no qualifying state refundable credit for this household, so the omitted output should have been 0." -us,scenario_064,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable value for state_refundable_credits. The required Wisconsin refundable-credit computation yields no qualifying state refundable credit for this household, so the omitted output should have been 0." -us,scenario_066,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated very low earnings as disqualifying for EITC, but the childless EITC phases in from the first dollars of earned income for an age-40 worker. It also confused lack of tax liability with lack of refundable credits; the EITC is refundable and equals 7.65% of $520, or $39.78." -us,scenario_066,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model correctly identified the 7.65% childless EITC phase-in and computed $39.78, then overrode that computation with an erroneous EITC table amount of $234. At $520 of earned income in the phase-in range, the credit is 0.0765 times earnings, not a plateau or expanded-law table amount." -us,scenario_066,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model imposed a nonexistent higher earned-income threshold for EITC eligibility. A childless age-40 worker with $520 of earned income qualifies during the phase-in range, generating a $39.78 refundable EITC." -us,scenario_066,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated the absence of qualifying children as eliminating EITC, but the childless-worker EITC applies to this 40-year-old filer. With $520 of wages, the correct phase-in calculation is 7.65% of earned income, yielding $39.78." -us,scenario_066,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model failed to count the childless EITC as a refundable federal credit at very low earnings. The household's $520 of wages is earned income in the EITC phase-in range, so refundable federal credits include $39.78 of EITC." -us,scenario_066,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model invented a minimum earned-income cutoff above $520 for positive EITC. For a childless age-40 worker, EITC phases in at 7.65% of earned income from low earnings, so $520 produces $39.78." -us,scenario_066,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated having no qualifying children as making the household ineligible for EITC. The childless-worker EITC applies to a 40-year-old single filer, and $520 of wages generates $39.78 in the phase-in range." -us,scenario_066,head_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model correctly recognized that $520 of annual income is below Virginia's ACA expansion income threshold, but then treated other means-tested health coverage at interview as a Medicaid eligibility exclusion. It also imported an asset concern into a MAGI adult expansion pathway, where PolicyEngine does not apply a bank-account asset test." -us,scenario_066,head_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model applied the correct Virginia adult expansion income pathway and found the $520 MAGI below 138% FPL, then incorrectly made other means-tested health coverage disqualifying. PolicyEngine's Medicaid eligibility output asks eligibility, not enrollment exclusivity, and that coverage fact does not override ACA adult expansion eligibility." -us,scenario_066,head_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model replaced the prompt's annual gross wages of $520 with an imputed full-year wage from hourly pay and usual weekly hours. PolicyEngine uses the listed annual wages for MAGI in this constant-year benchmark, so MAGI remains about 0.03 x FPL and below Virginia's adult expansion limit." -us,scenario_066,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model missed Virginia's ACA adult expansion category for adults aged 19-64. A 40-year-old non-dependent adult with MAGI far below the expansion limit is in the ADULT Medicaid category even without disability, SSI, pregnancy, or child status." -us,scenario_066,head_medicaid_eligible,gpt-5.4-mini,llm_error,health_coverage,False,"The model treated other means-tested health coverage at interview as a Medicaid eligibility bar. Under the PolicyEngine rule applied here, that fact does not negate eligibility through Virginia's MAGI-based ACA adult expansion category." -us,scenario_066,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model failed to recognize the ACA adult expansion eligibility category as the qualifying pathway. A 40-year-old Virginia adult with MAGI at 0.03 x FPL qualifies through the ADULT category without needing a separate disability or health-status indicator. -us,scenario_066,head_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"The model made other means-tested health coverage at interview a preclusion from Medicaid eligibility. PolicyEngine's Virginia adult expansion determination remains based on age, immigration status, non-dependent status, and MAGI below the threshold, all of which are satisfied." -us,scenario_066,head_medicaid_eligible,minimax-m3,llm_error,asset_resource,False,"The model applied a $2,000 asset limit to Virginia Medicaid adult expansion. PolicyEngine's MAGI-based ACA adult category does not use the $14,000 bank balance as an asset test, and the $520 MAGI is below the expansion income limit." -us,scenario_066,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model annualized hourly wage and usual hours to substitute about $66,560 for the listed annual wages of $520, then compounded the error by treating other means-tested health coverage as disqualifying. PolicyEngine uses the stated annual wage amount for MAGI and does not use the other-coverage interview fact to block Virginia adult expansion eligibility." -us,scenario_066,snap,claude-fable-5,llm_error,period_annualization,False,"The model held the one-person maximum allotment fixed at $298 for all 12 months. PolicyEngine applies the later-year increase to $304.68 and sums the monthly values, yielding $3,596.04." -us,scenario_066,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied the ordinary SNAP resource limit to the $14,000 bank balance. The household qualifies categorically through TANF non-cash assistance, so that balance does not bar SNAP eligibility." -us,scenario_066,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $275 monthly maximum allotment instead of the applicable 2026 monthly values. With net income reduced to zero, the benefit equals the full monthly maximum, including the later increase to $304.68." -us,scenario_066,snap,claude-opus-4.8,llm_error,other,False,"The model correctly reduced net income to essentially zero but then subtracted an unexplained income offset and submitted $2,496. Zero SNAP net income requires the full maximum allotment each month, whose 12-month sum is $3,596.04." -us,scenario_066,snap,claude-opus-5,llm_error,thresholds_rates,False,The model used an estimated maximum of $283 per month. The applicable monthly maximum is $298 for most months and $304.68 after the midyear guideline change. -us,scenario_066,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model treated Virginia categorical eligibility as unavailable unless separately stated as TANF or SSI receipt and therefore imposed the ordinary resource limit. PolicyEngine grants categorical eligibility through TANF non-cash assistance, which removes the $14,000 asset barrier." -us,scenario_066,snap,claude-sonnet-5,llm_error,period_annualization,False,"The model misread the explicitly annual $520 wage amount as $520 per week. The correct monthly gross income is $43.33, which passes the SNAP income tests." -us,scenario_066,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model limited benefits to three months by declaring that the head failed the ABAWD work requirement. The stated 40 usual weekly hours satisfies the 20-hour work standard, so all 12 eligible months must be counted." -us,scenario_066,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model substituted an estimated $291 monthly maximum for the applicable 2026 allotments. PolicyEngine uses $298 for most months and $304.68 later in the year. -us,scenario_066,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model returned zero without applying the eligibility pathways in the facts. Gross income is $43.33 monthly, categorical eligibility through TANF non-cash assistance neutralizes the asset test, and zero net income produces the full monthly allotment." -us,scenario_066,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly applied categorical eligibility but used $291 as a constant monthly maximum. The applicable allotment is $298 for most months and rises to $304.68 later in 2026. -us,scenario_066,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a constant $291 monthly maximum. The 2026 calculation uses $298 in most months and $304.68 in later months, totaling $3,596.04." -us,scenario_066,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,The model used a constant $292 monthly allotment. Zero net income entitles the household to the applicable maximum of $298 in most months and $304.68 after the guideline update. -us,scenario_066,snap,glm-5.2,llm_error,thresholds_rates,False,"The model rounded the maximum allotment to approximately $300 and multiplied it by 12. The calculation requires the exact monthly values of $298 and $304.68, whose annual sum is $3,596.04." -us,scenario_066,snap,gpt-5.4-mini,llm_error,asset_resource,False,"The model applied a typical SNAP resource limit to the $14,000 bank balance. Categorical eligibility through TANF non-cash assistance makes the household eligible without that ordinary asset test." -us,scenario_066,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model defaulted the benefit to zero instead of evaluating the stated eligibility facts. The household passes the income tests, receives categorical eligibility through TANF non-cash assistance, satisfies the remaining tests, and therefore receives positive SNAP in every month." -us,scenario_066,snap,gpt-5.5,llm_error,period_annualization,False,"The model fixed the maximum allotment at $298 for the entire year. The maximum increases to $304.68 in later months, so the monthly allotments sum to $3,596.04 rather than $3,576." -us,scenario_066,snap,gpt-5.6-luna,llm_error,period_annualization,False,The model multiplied the $298 maximum by 12 and omitted the later-year adjustment. PolicyEngine applies $304.68 in the later months and sums all monthly allotments. -us,scenario_066,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model correctly derived zero net income but assumed a constant $298 maximum for all 12 months. The later monthly maximum is $304.68, producing an annual total of $3,596.04." -us,scenario_066,snap,gpt-5.6-terra,llm_error,asset_resource,False,"The model imposed the standard resource limit on the $14,000 bank balance. TANF non-cash categorical eligibility prevents that balance from disqualifying the household." -us,scenario_066,snap,grok-4.3,llm_error,asset_resource,False,"The model applied a typical asset limit and treated the separate health-coverage fact as adverse to SNAP. Categorical eligibility through TANF non-cash assistance removes the ordinary asset barrier, and other means-tested health coverage does not eliminate SNAP eligibility." -us,scenario_066,snap,grok-4.5,llm_error,thresholds_rates,False,The model correctly waived the asset test and derived zero net income but used $292 as the maximum allotment. The applicable values are $298 for most months and $304.68 later in 2026. -us,scenario_066,snap,grok-build-0.1,llm_error,thresholds_rates,False,The model used a projected $292 monthly maximum rather than the applicable monthly allotments. The full benefit is $298 in most months and $304.68 in later months. -us,scenario_066,snap,kimi-k2.6,llm_error,asset_resource,False,"The model required the prompt to list a categorical-eligibility waiver explicitly and then applied the standard resource limit. PolicyEngine establishes categorical eligibility through TANF non-cash assistance, so the $14,000 bank balance does not disqualify the household." -us,scenario_066,snap,kimi-k3,llm_error,period_annualization,False,"The model held the maximum allotment at $298 for all 12 months. It omitted the later-year increase to $304.68 that raises the annual sum to $3,596.04." -us,scenario_066,snap,minimax-m3,llm_error,asset_resource,False,"The model treated the $14,000 bank balance as disqualifying under the ordinary resource limit. TANF non-cash categorical eligibility removes that asset test for this household." -us,scenario_066,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model first recognized very low income but then subtracted an unexplained $872 annual income contribution. The 20% earned-income deduction and standard deduction reduce SNAP net income to zero, so the 30% contribution is zero and the household receives the full monthly maximum." -us,scenario_066,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model applied an outdated 15% Virginia refundable-EITC rate to the federal EITC. The 2026 rate is 20%, so $39.78 × 20% rounds to $7.96 rather than $6.00." -us,scenario_066,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly treated Virginia’s EITC as exclusively nonrefundable and denied the refundable state EITC. In 2026 the household can claim a refundable credit equal to 20% of its $39.78 federal EITC. -us,scenario_066,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model invented a minimum-earnings threshold and set the childless federal EITC to zero. The EITC phases in from the first dollar of earnings, producing $39.78 federally and a $7.96 Virginia refundable EITC." -us,scenario_066,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model wrongly rounded a positive refundable credit to zero and also used a 15% state rate. Virginia applies its 20% refundable-EITC rate to the $39.78 federal EITC, yielding $7.96 regardless of the standard deduction or zero tax liability." -us,scenario_066,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated Virginia’s EITC option as nonrefundable and limited it to tax liability. The elected refundable pathway pays 20% of the $39.78 federal EITC, producing $7.96 even when pre-credit state tax is zero." -us,scenario_066,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model imposed a nonexistent roughly $600 minimum earned-income threshold for the childless federal EITC. At $520 of earnings the federal credit is $39.78, and Virginia refunds 20% of it, or $7.96." -us,scenario_066,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model ignored the prompt’s instruction to assume filing and program take-up when required, treating the refundable-EITC election as absent. With that election assumed, Virginia refunds 20% of the $39.78 federal EITC, or $7.96." -us,scenario_066,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model classified Virginia’s EITC as solely nonrefundable. Virginia’s refundable EITC pathway applies here and pays 20% of the $39.78 federal credit, yielding $7.96." -us,scenario_066,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly identified the federal EITC base but applied a 15% Virginia rate. The 2026 refundable rate is 20%, so $39.78 produces $7.96 rather than $5.97." -us,scenario_066,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The answer omitted the household’s qualifying Virginia refundable EITC. Applying the 20% Virginia rate to the $39.78 federal EITC yields $7.96. -us,scenario_066,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a 15% Virginia refundable-EITC rate and rounded the federal base before applying it. The applicable rate is 20% of the unrounded $39.78 federal EITC, which rounds to $7.96." -us,scenario_066,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a 15% Virginia refundable-EITC rate to the correct $39.78 federal base. The 2026 rate is 20%, yielding $7.96." -us,scenario_066,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used the wrong Virginia refundable-EITC percentage. Applying the 2026 rate of 20%, rather than 15%, to $39.78 gives $7.96." -us,scenario_066,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly made zero state tax liability disqualifying and omitted the refundable EITC. Refundability permits a payment beyond liability, and 20% of the $39.78 federal EITC is $7.96." -us,scenario_066,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to trigger Virginia’s refundable EITC despite the household’s federal EITC eligibility and Virginia residency. The state credit is 20% of $39.78, which rounds to $7.96." -us,scenario_066,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly assumed that no refundable state credit applies at this income level. The low earnings generate a $39.78 federal EITC, which supports a $7.96 Virginia refundable EITC." -us,scenario_066,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model applied a 15% Virginia rate to an approximate federal EITC. Virginia’s 2026 refundable rate is 20% of the $39.78 federal EITC, producing $7.96." -us,scenario_066,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model denied the refundable EITC merely because the filer is childless and has $520 of wages. A qualifying child is not required for this 40-year-old filer’s federal EITC, and Virginia refunds 20% of the resulting $39.78 credit, or $7.96." -us,scenario_066,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model treated being childless as eliminating Virginia’s refundable credit. The head qualifies for the childless federal EITC, and Virginia’s 20% refundable credit equals $7.96." -us,scenario_066,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model conditioned refundable credits on positive state tax liability. Virginia’s refundable EITC is payable beyond liability and equals 20% of the $39.78 federal EITC, or $7.96." -us,scenario_066,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model omitted the Virginia refundable EITC available to this low-income single adult. The $39.78 federal EITC generates a $7.96 state refundable credit at the 20% rate. -us,scenario_066,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Virginia has no applicable refundable state EITC. The refundable pathway applies in 2026 and pays 20% of the household’s $39.78 federal EITC, yielding $7.96." -us,scenario_066,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated having no dependents and no state tax liability as disqualifying. The head qualifies for the childless federal EITC, and Virginia refunds 20% of that $39.78 credit, producing $7.96 beyond tax liability." -us,scenario_066,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model incorrectly ended Virginia’s refundable-EITC provision after 2025. The refundable credit applies in 2026 at 20% of the $39.78 federal EITC, yielding $7.96." -us,scenario_066,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model omitted the applicable Virginia refundable EITC. Federal EITC eligibility produces a $39.78 base, and the 20% Virginia credit rounds to $7.96." -us,scenario_066,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly required positive tax liability or a qualifying child for Virginia’s refundable EITC. The childless head qualifies for $39.78 of federal EITC, and the refundable state option pays 20% of it, or $7.96, despite zero state liability." -us,scenario_067,dependent1_medicaid_eligible,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly included the parents’ wages and pension in dependent1’s ACA adult expansion MAGI and then tested a three-person household against 138% FPL. PolicyEngine’s category calculation gives dependent1 a MAGI income level of 0.00 times FPL, so parental income and resources do not defeat this pathway." -us,scenario_067,dependent1_medicaid_eligible,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model treated dependent1’s tax-unit-dependent status as requiring the parents’ income to be counted in the ACA adult expansion test. The applicable calculation assigns dependent1 MAGI of 0.00 times FPL, which is below Indiana’s 138% FPL limit." -us,scenario_067,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model substituted total household income for dependent1’s MAGI income level. Dependent1’s applicable MAGI is 0.00 times FPL, satisfying Indiana’s ACA adult expansion income test." -us,scenario_067,dependent1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model assumed that living with and being claimed by the parents makes their combined income controlling for dependent1’s Medicaid test. PolicyEngine’s adult-category calculation instead assigns dependent1 MAGI of 0.00 times FPL, below the expansion threshold." -us,scenario_067,dependent1_medicaid_eligible,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model used the tax unit’s $87,302 of wages and pension as dependent1’s Medicaid income. The relevant adult expansion calculation gives dependent1 MAGI of 0.00 times FPL, not the tax unit’s aggregate income." -us,scenario_067,dependent1_medicaid_eligible,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added the parents’ earnings and pension plus dependent1’s disability benefits and tested the resulting $98,102 against a three-person 138% FPL threshold. The applicable MAGI calculation for dependent1 is 0.00 times FPL, so that household-income aggregation is the wrong computation." -us,scenario_067,dependent1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model declined to recognize eligibility because SSI receipt or separate Medicaid-qualification details were not listed. Dependent1 qualifies through Indiana’s ACA adult expansion category based on age under 65 and MAGI of 0.00 times FPL; SSI is not required for that pathway. -us,scenario_067,dependent1_medicaid_eligible,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model counted the parents’ income in dependent1’s MAGI test and incorrectly required SSI enrollment or another categorical pathway. Indiana’s ACA adult expansion pathway applies directly, and dependent1’s calculated MAGI is 0.00 times FPL." -us,scenario_067,dependent1_medicaid_eligible,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model compared total household income with Indiana’s Medicaid limits instead of using dependent1’s applicable MAGI income level. That level is 0.00 times FPL, below the 138% adult expansion threshold." -us,scenario_067,dependent1_medicaid_eligible,gpt-5.6-terra,llm_error,asset_resource,False,"The model treated household resources and aggregate income as disqualifying across all Medicaid pathways. Dependent1 is tested under the MAGI-based ACA adult expansion category, where the engine calculates 0.00 times FPL and does not apply the listed household assets as an eligibility bar." -us,scenario_067,dependent1_medicaid_eligible,grok-4.3,llm_error,asset_resource,False,"The model treated dependent1’s disability benefits and household resources as exceeding the controlling Medicaid thresholds. The controlling ACA adult expansion test is MAGI-based, assigns dependent1 0.00 times FPL, and does not disqualify dependent1 based on the listed household resources." -us,scenario_067,dependent1_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model falsely characterized Indiana as a non-expansion state and analyzed dependent1 chiefly under a disabled-adult pathway using parental income. Indiana has adopted ACA Medicaid expansion, and dependent1 qualifies in its adult category with MAGI of 0.00 times FPL." -us,scenario_067,dependent1_medicare_eligible,claude-opus-5,llm_error,age_disability,False,The model treated disability and receipt of unspecified disability benefits as sufficient for Medicare. It failed to require a specified qualifying entitlement pathway and completion of the applicable waiting period for this 23-year-old. -us,scenario_067,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,The model inferred that the unspecified disability benefits were SSDI and then invented satisfaction of the 24-month Medicare waiting period from the instruction that listed facts remain constant during the year. Constant annual status does not establish SSDI entitlement or the required duration of entitlement. -us,scenario_067,dependent1_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,The model relabeled unspecified disability benefits as SSDI and treated disabled benefit receipt as proof of Medicare eligibility. It omitted the required evidence of SSDI entitlement and completion of the disability-based waiting period. -us,scenario_067,dependent1_medicare_eligible,gpt-5.5,llm_error,age_disability,False,The model used the disabled flag plus annual unspecified disability benefits as a direct Medicare-eligibility test. Neither fact establishes the qualifying SSDI entitlement and waiting-period conditions required for an under-65 person. -us,scenario_067,dependent1_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model incorrectly asserted that PolicyEngine treats disability and unspecified disability benefits as disability-based Medicare eligibility. The dependent is under 65, and no qualifying Medicare entitlement status or completed waiting period was supplied." -us,scenario_067,dependent1_medicare_eligible,kimi-k3,llm_error,age_disability,False,The model assumed that PolicyEngine converts the disabled flag and unspecified disability-benefit receipt into Medicare eligibility. It failed to distinguish general disability status from a specified under-65 Medicare entitlement pathway. -us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $1,845 of auto-loan interest and used a $32,600 standard deduction instead of $32,200. Its submitted $3,391 also contradicts its own final calculation of $5,347, so it failed to carry even its mistaken computation into the output." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $28,500 standard deduction instead of $32,200 and then invented approximately $1,266 of nonrefundable credits. The only applicable nonrefundable credit in the trace is the $500 Credit for Other Dependents." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model’s stated derivation reaches approximately $5,620 after the $500 dependent credit, essentially the correct computation, but it submitted $6,280. It failed to transfer its own calculated result into the required value." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the mortgage balance and used that unsupported amount in an itemized-deduction analysis. It then submitted $5,594 despite its own itemized calculation producing about $6,722, making the final value inconsistent with its reasoning." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model explicitly derived about $6,116 before credits and about $5,616 after the $500 Credit for Other Dependents, then submitted $6,104. It failed to apply its own stated nonrefundable-credit result to the output." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a projected $30,000 standard deduction and estimated tax-bracket thresholds instead of the 2026 $32,200 deduction and applicable brackets. That overstated taxable income by $2,200 and produced tax after the $500 dependent credit of $5,886 rather than $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model’s arithmetic produced roughly $5,568 after the $500 dependent credit, but it submitted $10,500 after invoking nonexistent self-employment adjustments and benefit phase-outs. Neither item affects this federal income-tax output." -us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed a post-TCJA reversion, replacing the $32,200 standard deduction and 12% bracket with a $16,500 deduction, personal exemptions, and a 15% bracket. It also omitted the applicable $500 Credit for Other Dependents." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $9,180 of the dependent’s disability benefits in the parents’ AGI and applied expired-law personal exemptions and 15% rates. It also invented a SALT deduction, whereas the trace uses $87,302 gross income, the $32,200 standard deduction, and a $500 dependent credit." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used approximately $60,400 of taxable income instead of $55,102 and treated the household as having no qualifying dependent credit. The disabled adult dependent generates a $500 Credit for Other Dependents after tax of $6,116.24." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied an assumed TCJA expiration, personal exemptions, itemized deductions, and a 15% bracket instead of the operative 2026 standard deduction and rates. It also failed to subtract the $500 Credit for Other Dependents." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used pre-TCJA personal exemptions and invented mortgage-interest and state-tax deductions from amounts not supplied as deductible expenses. The correct computation takes the $32,200 standard deduction and then subtracts the $500 Credit for Other Dependents." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $30,800 standard deduction and 2025 bracket thresholds instead of the 2026 $32,200 deduction and applicable brackets. Its treatment of the $500 dependent credit was correct, but taxable income should be $55,102." -us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model substituted a $30,000 standard deduction and a 2025 bracket proxy for the 2026 parameters. Using the $32,200 deduction produces $55,102 of taxable income, $6,116.24 before credits, and $5,616.24 after the $500 dependent credit." -us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The submitted $4,567 does not follow from the stated wages, pension, $32,200 standard deduction, and 2026 brackets. Those inputs yield $6,116.24 before credits, and the applicable $500 Credit for Other Dependents yields $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked unspecified itemized deductions from the mortgage and medical context instead of using the larger $32,200 standard deduction. The trace yields $55,102 of taxable income and only a $500 nonrefundable dependent credit, producing $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $9,180 of the adult dependent’s disability benefits to joint taxable income. Those benefits are not part of the filers’ $87,302 gross income, so taxable income after the $32,200 standard deduction is $55,102, not $64,282." -us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The submitted $8,500 is inconsistent with applying the 2026 married-joint standard deduction and brackets to the stated $87,302 of gross income. The resulting regular tax is $6,116.24, reduced by the $500 Credit for Other Dependents." -us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied personal exemptions and itemized medical and SALT deductions under an assumed post-TCJA reversion. It should use the $32,200 standard deduction and subtract the $500 Credit for Other Dependents rather than assigning zero nonrefundable credits." -us,scenario_067,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied a reverted $17,291 standard deduction and 15% bracket instead of the operative $32,200 deduction and 12% bracket at this income. It also omitted the disabled adult dependent’s $500 Credit for Other Dependents." -us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly excluded the dependent’s benefits and applied the $500 dependent credit, but used an estimated $30,800 standard deduction and estimated bracket threshold. The 2026 $32,200 deduction leaves $55,102 taxable income and produces $5,616.24 after the credit." -us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $9,180 of the dependent’s disability benefits in the joint AGI and also deducted $1,845 of auto-loan interest. The filers’ gross income is $87,302 with no such auto-interest deduction, leaving $55,102 taxable income after the standard deduction." -us,scenario_067,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model included the dependent’s $10,800 disability benefits in the parents’ AGI and granted a $2,000 Child Tax Credit. A 23-year-old disabled dependent does not meet the CTC under-17 requirement; the applicable nonrefundable credit is the $500 Credit for Other Dependents." -us,scenario_067,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the dependent’s $10,800 disability benefits to the filers’ gross income and used a projected $31,200 standard deduction. The benefits remain the dependent’s income, so the filers use $87,302 of gross income and a $32,200 standard deduction before applying the $500 dependent credit." -us,scenario_067,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly treated the 23-year-old disabled dependent as a qualifying child for the refundable Additional Child Tax Credit. Disability does not make a 23-year-old eligible for the CTC child amount, so the $1,700 ACTC shortcut does not apply and the refundable CTC component is $0." -us,scenario_067,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model applied an incorrect Medicare age threshold by treating age 60 as old enough for Medicare. The head is 60, below the age-65 Medicare threshold, and no alternative Medicare eligibility pathway is listed for the head." -us,scenario_067,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly computed $6,300.54 from the head's wages, then abandoned the prompt rule that unlisted numeric inputs are 0 and inferred spouse earnings from weekly hours. Its submitted $12,554.41 reflects adding payroll tax for an invented wage earner instead of limiting employee FICA to the head's listed $82,360 of wages." -us,scenario_067,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model used the correct FICA rates and wage base treatment but rounded the two component taxes to whole dollars before summing. Social Security is $5,106.32 and Medicare is $1,194.22, so the annual payroll tax is $6,300.54 rather than $6,299.00." -us,scenario_067,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model identified the right taxable wage earner and tax types but miscomputed the FICA arithmetic. Applying 6.2% and 1.45% to $82,360 yields $5,106.32 of Social Security tax plus $1,194.22 of Medicare tax, not $5,727.14." -us,scenario_067,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model used the correct FICA rates and taxable wages but replaced the exact calculation with a broad rounded estimate. The exact annual employee-side tax is $5,106.32 plus $1,194.22, so rounding to $6,376 overstates the required output." +us,scenario_064,ssi,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated Dependent 1's generic disability flag as satisfying SSI's program-specific aged, blind, or disabled criterion. Dependent 1 has `is_ssi_aged_blind_disabled=False`, so applying the one-third in-kind-support reduction and a federal benefit rate was invalid; SSI is $0 before those calculations arise." +us,scenario_064,ssi,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model equated the listed generic disability status with a qualifying SSI disability determination and awarded the maximum federal benefit. Dependent 1 fails `is_ssi_aged_blind_disabled`, so personal income, assets, and the federal benefit rate never affect the result." +us,scenario_064,ssi,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly classified Dependent 1 as categorically eligible for SSI from the generic disability fact, then added a Wisconsin supplement. Because Dependent 1 has `is_ssi_aged_blind_disabled=False`, neither the federal SSI payment nor a state supplement is payable." +us,scenario_064,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model treated age 27 plus the generic disability flag as sufficient SSI categorical eligibility and proceeded to parental-deeming and COLA calculations. Dependent 1 fails the SSI-specific aged, blind, or disabled test, so deeming, income, resources, and the projected federal benefit rate are irrelevant and SSI is $0." +us,scenario_064,ssi,gpt-5.5,llm_error,categorical_eligibility,False,"The model assumed the disabled adult dependent satisfied SSI's disability criterion and multiplied the maximum individual rate by 12. Dependent 1 has `is_ssi_aged_blind_disabled=False`, so the maximum-rate computation does not apply." +us,scenario_064,ssi,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model converted the generic disability fact directly into SSI categorical eligibility and awarded the full federal benefit rate. Dependent 1 fails the SSI-specific aged, blind, or disabled criterion, producing $0 regardless of personal income or assets." +us,scenario_064,ssi,inkling,llm_error,categorical_eligibility,False,"The model incorrectly admitted Dependent 1 into SSI based on the generic disability flag, then applied a one-third living-arrangement reduction and an estimated Wisconsin supplement. Dependent 1 has `is_ssi_aged_blind_disabled=False`, so neither the living-arrangement adjustment nor the supplement is reached." +us,scenario_064,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract." +us,scenario_064,ssi,kimi-k3,llm_error,categorical_eligibility,False,"The model declared Dependent 1 categorically SSI-eligible from the generic disability fact and then applied the full individual federal rate after excluding parental deeming. Dependent 1 instead fails `is_ssi_aged_blind_disabled`, so deeming, resources, and the benefit rate never enter the computation." +us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model calculated tax near $4,964 but then reduced it to $2,699 through unsupported Wisconsin adjustments and credits, despite identifying only an approximately $265 itemized-deduction credit. It also used an inflated income base and standard deduction instead of $114,403.37 of AGI and the $8,759.12 Wisconsin deduction." +us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model started from an approximate $104,434 federal-taxable-income figure rather than Wisconsin AGI of $114,403.37 and never performed the traced Wisconsin deduction-and-exemption calculation. The correct Wisconsin taxable income is $102,144.25, which produces $4,606." +us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model computed roughly $5,026 of bracket tax and then subtracted an invented $875 of school-property and itemized-deduction credits. No property tax was listed, and its assumed mortgage interest and medical credit were not established by the inputs; the traced pre-refundable liability is $4,606." +us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used an approximate $113,000 Wisconsin income base and a $9,500 standard deduction instead of AGI of $114,403.37 and the exact $8,759.12 deduction. It also treated personal exemptions inconsistently as a credit-equivalent adjustment, preventing the correct $102,144.25 taxable-income calculation." +us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model rounded the bracket result to $4,600 instead of carrying the Wisconsin computation through to the requested amount. Taxable income of $102,144.25 under the 2026 joint schedule yields $4,606." +us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used an erroneous $3,957 Wisconsin standard deduction, producing taxable income of $108,583 instead of $102,144.25. It then invented $520 of unspecified nonrefundable credits to force the result downward." +us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double counted the $14,000 self-employment partnership amount in addition to partnership/S-corporation income and incorrectly deducted the $1,796 domestic-production amount and retirement contributions in constructing AGI. Wisconsin AGI is $114,403.37, followed by an $8,759.12 standard deduction and $3,500 of exemptions." +us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted an inapplicable flat $25,000 standard deduction and the domestic-production deduction from its AGI. Wisconsin's income-dependent standard deduction is $8,759.12, leaving $102,144.25 taxable rather than $82,566." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated the Wisconsin standard deduction as $13,506, lowering taxable income to $97,576 instead of $102,144.25. It also subtracted an unsupported $150 dependent credit; five dependents-and-filers are already reflected through $3,500 of personal exemptions." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied only a generic bracket estimate and did not calculate from $114,403.37 of Wisconsin AGI, the $8,759.12 standard deduction, and $3,500 of exemptions. Those steps yield $102,144.25 of taxable income and $4,606 of tax." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated Wisconsin AGI as $92,978 rather than $114,403.37, causing taxable income to fall to about $78,969. The traced Wisconsin deductions leave $102,144.25 taxable." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated Wisconsin taxable income as $82,256 instead of $102,144.25 by misapplying the sliding standard deduction. The correct deduction is $8,759.12, followed by $3,500 of personal exemptions." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The unexplained $4,125 estimate does not implement the traced Wisconsin calculation. AGI of $114,403.37 less $8,759.12 and $3,500 produces $102,144.25 taxable and $4,606 of tax." +us,scenario_064,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model incorrectly treated deductions and unspecified household circumstances as eliminating Wisconsin tax. After the applicable standard deduction and exemptions, $102,144.25 remains taxable, and no nonrefundable credits reduce the resulting $4,606 to zero." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model assumed unspecified deductions and credits exceeded the Wisconsin liability. The applicable deductions leave $102,144.25 taxable, and the resulting $4,606 is not erased by any listed nonrefundable credit." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model overstated the Wisconsin standard deduction at $15,166 and counted only two $700 exemptions instead of five. It then subtracted an unsupported $472 medical/itemized-deduction credit; the traced deductions produce $102,144.25 taxable and $4,606 of liability." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The unexplained $2,800 estimate omits the actual Wisconsin income-base and bracket computation. The applicable deductions reduce $114,403.37 of AGI only to $102,144.25 of taxable income, which yields $4,606." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model named the correct calculation structure but produced $4,872, which is inconsistent with applying the 2026 joint schedule to $102,144.25. The schedule yields $4,606 after the $8,759.12 standard deduction and $3,500 of exemptions." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model treated Wisconsin personal exemptions as credits and did not show the income-dependent standard deduction. They are deductions from AGI totaling $12,259.12, producing $102,144.25 of taxable income and $4,606 of tax." +us,scenario_064,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model rounded a generic Wisconsin tax estimate to $4,500 without calculating the traced taxable income. Applying the 2026 brackets to $102,144.25 yields $4,606." +us,scenario_064,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated the Wisconsin standard deduction at about $6,500 and consequently overstated taxable income at about $107,900. The deduction is $8,759.12 and, after $3,500 of exemptions, taxable income is $102,144.25." +us,scenario_064,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used 2024 Wisconsin brackets and obsolete 4.75% and 5.75% rates instead of the 2026 schedule. It also started from an understated AGI and an overstated standard deduction rather than the traced $114,403.37 and $8,759.12 figures." +us,scenario_064,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model understated the starting income at about $97,723 and treated the overtime provision as a Wisconsin AGI addback rather than following the engine's $114,403.37 AGI. The correct deductions leave $102,144.25 taxable, not approximately $90,000." +us,scenario_064,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. +us,scenario_064,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model applied a nonexistent 6.27% Wisconsin bracket to income around $107,800. At $102,144.25 of 2026 joint taxable income, the applicable upper marginal rate is 5.3%, producing $4,606." +us,scenario_064,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer implies that the model treated deductions or credits as eliminating all Wisconsin taxable income without performing the calculation. The applicable deductions leave $102,144.25 taxable and generate $4,606 of liability." +us,scenario_064,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model improperly excluded the $13,899 overtime premium from wages when constructing federal AGI and then used an overstated $13,500 Wisconsin standard deduction. Wisconsin AGI is $114,403.37 and the state deduction is $8,759.12, yielding $102,144.25 taxable." +us,scenario_064,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model deducted $11,827 of retirement contributions and medical expenses directly from Wisconsin taxable income even though those deductions are not supported by the traced state calculation. The applicable reductions are the $8,759.12 Wisconsin standard deduction and $3,500 of personal exemptions, leaving $102,144.25 taxable." +us,scenario_064,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly ruled out every Wisconsin refundable-credit pathway it identified, including the homestead credit, Wisconsin earned income credit, farmland preservation credit, and veterans property tax credit, but then inserted an unexplained $154 residual. With every applicable component equal to zero, the required aggregation yields $0." +us,scenario_064,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required structured-output contract." +us,scenario_064,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required structured-output contract." +us,scenario_066,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a substantial-earned-income minimum for the EITC and therefore excluded a childless worker with $520 of wages. The childless EITC begins phasing in from the first dollar of earned income, producing 7.65% × $520 = $39.78." +us,scenario_066,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model correctly calculated the phase-in formula as 7.65% × $520 = $39.78, then discarded that result and substituted an erroneous $234 table value. Neither an EITC plateau nor table rounding applies at $520 of earnings in a way that changes the phase-in credit to $234." +us,scenario_066,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly imposed a higher minimum-earned-income requirement for the EITC. At $520 of wages, the childless worker is eligible and receives the phase-in amount of 7.65% × $520 = $39.78." +us,scenario_066,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated very low earnings and the absence of children as disqualifying for the EITC. A 40-year-old worker without qualifying children can claim the childless EITC, and $520 of earnings generates $39.78 in the phase-in range." +us,scenario_066,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,The model set all refundable credits to zero without applying the childless EITC phase-in formula. The applicable 7.65% rate on $520 of earned income produces $39.78 even though the other refundable credits are zero. +us,scenario_066,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model invented a minimum earned-income threshold above $520 for a positive EITC. The childless EITC phases in at 7.65% from the first dollar of earnings, so $520 yields $39.78." +us,scenario_066,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly concluded that $520 of wages and no qualifying children make the worker ineligible for EITC. The childless-worker pathway applies at age 40, and its 7.65% phase-in produces a $39.78 credit." +us,scenario_066,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly denied the childless EITC solely because earnings were $520 and there were no qualifying children. The taxpayer meets the childless-worker pathway, and 7.65% of $520 yields $39.78." +us,scenario_066,head_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model correctly identified that $520 is below Virginia's adult-expansion income limit but then wrongly treated other means-tested health coverage and $14,000 of assets as barriers. Neither current coverage nor assets disqualifies an otherwise eligible MAGI-based expansion adult." +us,scenario_066,head_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,The model invented a rule that other means-tested health coverage disqualifies an adult from Medicaid. The head instead qualifies through Virginia's ACA adult expansion category because the stated annual MAGI is only 0.03 times FPL. +us,scenario_066,head_medicaid_eligible,claude-sonnet-5,llm_error,period_annualization,False,"The model replaced the explicitly stated $520 annual wages with $32 × 40 × 52 and falsely asserted that PolicyEngine uses the higher amount. The prompt makes gross wages the annual total and demographic and work facts constant without authorizing wage imputation, so the MAGI test uses the low stated annual income." +us,scenario_066,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model missed Virginia's ACA adult expansion category, which covers qualifying adults ages 19–64 without requiring disability, parenthood, or another mandatory-category indicator. At age 40 and 0.03 times FPL, the head satisfies that pathway." +us,scenario_066,head_medicaid_eligible,gpt-5.4-mini,llm_error,health_coverage,False,"The model treated other means-tested health coverage as a Medicaid eligibility exclusion. That coverage fact does not block Virginia ACA adult expansion eligibility, which the head satisfies at 0.03 times FPL." +us,scenario_066,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model required a disability or other special categorical indicator and failed to apply Virginia's ACA adult expansion pathway. A 40-year-old adult with MAGI at 0.03 times FPL qualifies without disability or another health-status condition. +us,scenario_066,head_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"The model incorrectly made other means-tested health coverage preclusive. It is not an exclusion from the Virginia ACA adult expansion category, under which the head qualifies based on age, status, and MAGI." +us,scenario_066,head_medicaid_eligible,minimax-m3,llm_error,asset_resource,False,"The model applied a $2,000 asset limit from non-MAGI Medicaid pathways to Virginia's MAGI-based ACA adult expansion category. That category has no asset test, so the $14,000 bank balance does not offset eligibility at 0.03 times FPL." +us,scenario_066,head_medicaid_eligible,qwen-3.7-max,llm_error,period_annualization,False,"The model improperly annualized $32 per hour at 40 hours per week into $66,560 despite the prompt explicitly defining $520 as the full-year gross wage total. It also wrongly treated other means-tested health coverage as disqualifying; using the stated annual income places the head at 0.03 times FPL and within Virginia's adult expansion category." +us,scenario_066,snap,claude-fable-5,llm_error,period_annualization,False,"The model held the $298 monthly maximum constant for all 12 months. It omitted the three later months at $304.68, which raise the annual total from $3,576 to $3,596.04." +us,scenario_066,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary SNAP resource limit despite the household's categorical eligibility through TANF non-cash assistance. That pathway prevents the $14,000 bank balance from disqualifying the household." +us,scenario_066,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an obsolete or unsupported $275 monthly maximum allotment. The applicable 2026 maxima are $298 for nine months and $304.68 for three months, producing $3,596.04." +us,scenario_066,snap,claude-opus-4.8,llm_error,other,False,"The model correctly found essentially zero net income and identified a maximum benefit near $298 monthly, then subtracted an unexplained amount to reach $2,496. With zero SNAP net income there is no 30% income contribution, so the full monthly maxima total $3,596.04." +us,scenario_066,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model used $283 as the one-person monthly maximum. The applicable maxima are $298 and, after the mid-year guideline update, $304.68, totaling $3,596.04 across the year." +us,scenario_066,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model denied Virginia categorical eligibility and applied the ordinary resource limit to the $14,000 bank balance. The household qualifies categorically through TANF non-cash assistance, so the assets do not bar SNAP." +us,scenario_066,snap,claude-sonnet-5,llm_error,period_annualization,False,"The model misread the explicitly annual $520 wage amount as $520 per week. The correct monthly gross income is $43.33, which passes the income tests and leads to the full annual allotment of $3,596.04." +us,scenario_066,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model imposed the ABAWD three-month limit after ignoring the listed 40 usual weekly work hours. Forty hours per week satisfies the work requirement, so benefits are payable for all 12 months." +us,scenario_066,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an estimated $291 monthly maximum instead of the applicable 2026 monthly amounts. Nine months at $298 and three months at $304.68 total $3,596.04." +us,scenario_066,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The answer treated low income or assets as producing no benefit without applying the traced eligibility pathway. TANF non-cash categorical eligibility prevents the assets from disqualifying the household, and zero net income yields the full $3,596.04 allotment." +us,scenario_066,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly applied categorical eligibility but used $291 as the monthly maximum. The applicable amounts are $298 for nine months and $304.68 for three months, totaling $3,596.04." +us,scenario_066,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a flat $291 monthly maximum. It needed to apply $298 for nine months and the updated $304.68 amount for three months, yielding $3,596.04." +us,scenario_066,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a flat $292 monthly maximum. The traced 2026 maxima are $298 for nine months and $304.68 for three months, so the annual benefit is $3,596.04." +us,scenario_066,snap,glm-5.2,llm_error,thresholds_rates,False,"The model rounded the maximum allotment to $300 per month and applied it uniformly. The exact monthly values are $298 for nine months and $304.68 for three months, totaling $3,596.04." +us,scenario_066,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model applied a typical SNAP resource limit to the $14,000 bank balance. It omitted categorical eligibility through TANF non-cash assistance, under which the household remains eligible and receives $3,596.04." +us,scenario_066,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model returned zero without applying the household's categorical-eligibility and net-income computations. TANF non-cash categorical eligibility, passing income tests, and zero net income produce the full annual allotment of $3,596.04." +us,scenario_066,snap,gpt-5.5,llm_error,period_annualization,False,"The model applied $298 in every month and missed the later increase to $304.68. Nine months at $298 plus three months at $304.68 equal $3,596.04." +us,scenario_066,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The model annualized a constant $298 monthly maximum. It omitted the three later months at $304.68, which increase the annual amount to $3,596.04." +us,scenario_066,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model correctly reduced SNAP net income to zero but held the $298 maximum constant for 12 months. Applying the $304.68 maximum in the final three months gives $3,596.04." +us,scenario_066,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model applied the ordinary resource limit to the $14,000 bank balance. TANF non-cash categorical eligibility prevents that balance from disqualifying the household." +us,scenario_066,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model treated the $14,000 bank balance as disqualifying and gave irrelevant weight to other health coverage. TANF non-cash categorical eligibility supplies the SNAP pathway, and health coverage does not eliminate SNAP eligibility." +us,scenario_066,snap,grok-4.5,llm_error,thresholds_rates,False,"The model correctly waived the asset test and found zero net income but used $292 as a flat monthly maximum. The applicable monthly maxima total $3,596.04, not $3,504." +us,scenario_066,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a flat $292 maximum and also described the standard deduction as $204 annually rather than a monthly deduction. Zero net income still results, but the correct monthly maxima—$298 and later $304.68—sum to $3,596.04." +us,scenario_066,snap,inkling,llm_error,thresholds_rates,False,"The model rounded the monthly maximum to approximately $300 and multiplied by 12. Exact month-specific allotments of $298 and $304.68 sum to $3,596.04." +us,scenario_066,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model required an explicitly listed waiver and applied the ordinary resource limit. PolicyEngine derives categorical eligibility through TANF non-cash assistance, so the $14,000 balance does not bar SNAP." +us,scenario_066,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly found eligibility and zero net income but used $298 for every month. The maximum rises to $304.68 for three months, bringing the annual benefit to $3,596.04." +us,scenario_066,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model applied the standard asset limit and treated $14,000 as disqualifying. It omitted categorical eligibility through TANF non-cash assistance, which allows the household to qualify." +us,scenario_066,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model subtracted an $872 annual income contribution despite recognizing only $520 of gross annual earnings. The 20% earned-income deduction and standard deduction reduce SNAP net income to zero, so no 30% contribution is subtracted and the full monthly maxima total $3,596.04." +us,scenario_066,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model applied the 30% contribution to gross annual income instead of monthly SNAP net income after the earned-income, standard, and shelter deductions. It also invented a $149 monthly minimum and used a $213 maximum; zero net income instead yields the full $3,596.04 annual allotment." +us,scenario_066,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model applied an outdated 15% Virginia refundable-EITC percentage. For 2026 the applicable percentage is 20%, so $39.78 × 20% rounds to $7.96, not $6." +us,scenario_066,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated Virginia's EITC as exclusively nonrefundable and ignored the refundable EITC available in 2026. The household can claim 20% of its $39.78 federal EITC, producing $7.96 despite having no state tax liability." +us,scenario_066,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model invented a minimum-earnings barrier and set the federal childless EITC to zero. The EITC phases in from the first dollar of earned income, yielding $39.78 at $520 of wages and therefore a $7.96 Virginia refundable EITC." +us,scenario_066,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,The model both applied 15% instead of Virginia's 2026 rate of 20% and improperly rounded a positive refundable credit to zero. The standard deduction and zero state liability do not eliminate the refundable EITC; $39.78 × 20% rounds to $7.96. +us,scenario_066,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model conflated Virginia's nonrefundable low-income-credit option with the separately available refundable EITC. The refundable option pays 20% of the $39.78 federal EITC, or $7.96, even when pre-credit state tax is zero." +us,scenario_066,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model incorrectly imposed an approximately $600 minimum earned-income threshold for the childless federal EITC. The credit phases in on $520 of wages to $39.78, and Virginia refunds 20% of that amount, yielding $7.96." +us,scenario_066,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The prompt expressly assumes filing and credit take-up when required, so the model improperly refused to apply the refundable-EITC election. Applying the assumed claim produces 20% of the $39.78 federal EITC, or $7.96." +us,scenario_066,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model incorrectly classified Virginia's EITC as solely nonrefundable. Virginia's 2026 refundable EITC equals 20% of this household's $39.78 federal EITC, producing $7.96." +us,scenario_066,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly derived the $39.78 federal EITC but multiplied it by an outdated 15% state percentage. Virginia's 2026 refundable percentage is 20%, making the credit $7.96." +us,scenario_066,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted the Virginia refundable EITC for which the household qualifies. The credit is 20% of the $39.78 federal EITC, which rounds to $7.96." +us,scenario_066,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model applied a 15% Virginia refundable-EITC percentage instead of the 20% rate in effect for 2026. Using the unrounded federal EITC of $39.78 gives $7.96 rather than $6. +us,scenario_066,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used 15% instead of Virginia's 2026 refundable-EITC rate of 20%. Multiplying the correctly identified $39.78 federal EITC by 20% yields $7.96. +us,scenario_066,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used an outdated 15% Virginia refundable-EITC percentage. The 2026 percentage is 20%, so the $39.78 federal EITC generates $7.96." +us,scenario_066,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model wrongly used zero state tax liability to exclude refundable credits. Virginia's refundable EITC can exceed liability and equals 20% of the household's $39.78 federal EITC, or $7.96." +us,scenario_066,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to apply the Virginia refundable EITC triggered by the household's positive federal EITC. The state credit is 20% of $39.78, which rounds to $7.96." +us,scenario_066,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly assumed that no refundable state credit applies at this low income. The low earnings generate a $39.78 federal EITC, and Virginia refunds 20% of it, producing $7.96." +us,scenario_066,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a 15% Virginia percentage and rounded the federal EITC before applying it. Virginia's 2026 refundable rate is 20%, and $39.78 × 20% rounds to $7.96." +us,scenario_066,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly excluded a childless single adult from Virginia's refundable EITC. This adult has a $39.78 federal childless EITC, of which Virginia refunds 20%, yielding $7.96." +us,scenario_066,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model treated the absence of qualifying children as disqualifying for Virginia's refundable EITC. The head qualifies through the federal childless EITC pathway, generating a $39.78 federal credit and a $7.96 Virginia credit." +us,scenario_066,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly made positive state tax liability a condition for receiving the refundable EITC. Refundability permits a payment at zero liability, and 20% of the $39.78 federal EITC is $7.96." +us,scenario_066,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model omitted the Virginia refundable EITC available to this low-income single adult. The household's $39.78 federal childless EITC generates a $7.96 state refundable credit at the 20% rate. +us,scenario_066,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Virginia has no applicable refundable state EITC. Virginia refunds 20% of this household's $39.78 federal EITC in 2026, resulting in $7.96." +us,scenario_066,state_refundable_credits,inkling,llm_error,thresholds_rates,False,The model used a 15% Virginia refundable-EITC rate instead of the 20% rate applicable in 2026. Applying 20% to the precise $39.78 federal EITC yields $7.96. +us,scenario_066,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model wrongly treated zero state tax liability and no dependents as eliminating all refundable Virginia credits. The head qualifies for the federal childless EITC, and Virginia refunds 20% of the $39.78 federal amount, or $7.96." +us,scenario_066,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model incorrectly ended Virginia's refundable EITC after tax year 2025. The refundable credit applies in 2026 at 20% of the federal EITC, producing $7.96 from the household's $39.78 federal credit." +us,scenario_066,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,The model omitted the applicable Virginia refundable EITC. This household's $39.78 federal childless EITC produces a state refundable credit of $7.96 at the 20% rate. +us,scenario_066,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly required state tax liability or qualifying children for Virginia's refundable EITC. The head qualifies through the federal childless EITC pathway, and the refundable state amount is 20% of $39.78, or $7.96." +us,scenario_066,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly treated a single-person, childless household as ineligible for Virginia's refundable EITC. The head receives a $39.78 federal childless EITC, which generates a $7.96 Virginia refundable credit at 20%." +us,scenario_067,dependent1_medicaid_eligible,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly included the parents’ wages and pension in the dependent’s Medicaid MAGI and then evaluated a three-person household against 138% FPL. The dependent’s engine-calculated MAGI income level is 0.00 times FPL, qualifying them through Indiana’s ACA adult expansion category; parental resources and SSI-related rules do not control that pathway." +us,scenario_067,dependent1_medicaid_eligible,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model treated membership in the parents’ tax unit as sufficient to include their income in the dependent’s Medicaid MAGI. The applicable adult-expansion calculation gives the dependent a MAGI income level of 0.00 times FPL, below Indiana’s 138% limit." +us,scenario_067,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model substituted total household income for the dependent’s Medicaid MAGI. The dependent’s MAGI income level is 0.00 times FPL, so the 23-year-old qualifies under Indiana’s ACA adult expansion pathway." +us,scenario_067,dependent1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model incorrectly attributed the parents’ combined income to the dependent merely because the dependent belongs to their tax household. PolicyEngine’s applicant-level MAGI calculation is 0.00 times FPL, which satisfies Indiana’s adult expansion limit." +us,scenario_067,dependent1_medicaid_eligible,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model used the parents’ $87,302 of wages and pension as the dependent’s countable Medicaid income. The adult-expansion test instead yields a MAGI income level of 0.00 times FPL for the dependent, below 138% FPL." +us,scenario_067,dependent1_medicaid_eligible,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model pooled the parents’ earnings and pension with the dependent’s disability benefits and tested the resulting $98,102 against a three-person expansion threshold. The relevant MAGI calculation for the dependent is 0.00 times FPL, making them eligible through Indiana’s ACA adult expansion category." +us,scenario_067,dependent1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model declined to apply Medicaid eligibility because SSI receipt or separate Medicaid-qualification details were not listed. SSI is unnecessary here: age 23 and MAGI income at 0.00 times FPL place the dependent directly in Indiana’s ACA adult expansion category. +us,scenario_067,dependent1_medicaid_eligible,gpt-5.5,llm_error,household_unit_or_filing_status,False,The model wrongly counted the parents’ income in the dependent’s expansion MAGI and treated SSI or another categorical pathway as necessary. The dependent qualifies directly under Indiana’s adult expansion category because their MAGI income level is 0.00 times FPL. +us,scenario_067,dependent1_medicaid_eligible,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model applied total household income to the dependent instead of the engine’s Medicaid MAGI calculation for that applicant. The dependent’s MAGI income level is 0.00 times FPL, below Indiana’s 138% adult-expansion threshold." +us,scenario_067,dependent1_medicaid_eligible,gpt-5.6-terra,llm_error,asset_resource,False,"The model treated household income and listed resources as disqualifying across all Medicaid pathways. Indiana’s MAGI-based ACA adult expansion pathway has no asset test, and the dependent’s relevant MAGI income level is 0.00 times FPL." +us,scenario_067,dependent1_medicaid_eligible,grok-4.3,llm_error,asset_resource,False,"The model applied the dependent’s disability benefits and household resources as though they controlled the ACA adult expansion test. The applicable pathway is MAGI-based with no resource test, and PolicyEngine calculates the dependent’s MAGI income level as 0.00 times FPL." +us,scenario_067,dependent1_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model falsely characterized Indiana as a non-expansion state and analyzed a disabled-adult pathway using parental income. Indiana has adopted ACA Medicaid expansion, and the 23-year-old qualifies in its adult category because their MAGI income level is 0.00 times FPL." +us,scenario_067,dependent1_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the $10,800 of disability benefits as countable MAGI that exceeds Indiana’s adult Medicaid limit. The engine’s MAGI computation for the dependent is zero, placing them at 0.00 times FPL and within the ACA adult expansion threshold." +us,scenario_067,dependent1_medicare_eligible,claude-opus-5,llm_error,age_disability,False,The model treated disabled status plus generic disability benefits as automatically establishing disability-based Medicare eligibility. Those facts do not establish SSDI entitlement and completion of the Medicare waiting period. +us,scenario_067,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,The model improperly inferred that the unspecified disability benefits were SSDI and then treated the instruction that facts remain constant during the year as proof that the statutory waiting period had elapsed. Neither SSDI entitlement nor its required duration was supplied. +us,scenario_067,dependent1_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,The model relabeled generic disability benefits as SSDI and assumed the Medicare waiting period was satisfied. Disabled status and an annual benefit amount alone do not establish the under-65 Medicare pathway. +us,scenario_067,dependent1_medicare_eligible,gpt-5.5,llm_error,age_disability,False,The model treated disability and receipt of generic disability benefits as sufficient for Medicare before age 65. It omitted the required evidence of SSDI entitlement and completion of the statutory waiting period or another qualifying condition. +us,scenario_067,dependent1_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model assumed PolicyEngine converts disabled status plus any disability-benefit income directly into Medicare eligibility. The dependent has no specified SSDI entitlement, qualifying entitlement duration, or end-stage renal disease." +us,scenario_067,dependent1_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model incorrectly asserted that PolicyEngine treats generic disability benefits as a Medicare-qualifying disability pathway. At age 23, the dependent lacks the specified SSDI entitlement and waiting-period history needed for that pathway." +us,scenario_067,dependent1_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,The model collapsed disabled status and generic disability benefits into automatic Medicare eligibility. It failed to require SSDI entitlement followed by the statutory waiting period or another independently qualifying condition. +us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The submitted $3,391 contradicts the model's own final derivation of $5,347. That derivation also wrongly deducted $1,845 of auto-loan interest and used a $32,600 standard deduction instead of $32,200." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used an obsolete $28,500 joint standard deduction instead of $32,200, then invented approximately $1,266 of unspecified nonrefundable credits. The applicable dependent credit is exactly $500." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly outlined a result near $5,620, including the $500 dependent credit, but submitted $6,280 without a supporting computation. Its final number contradicts its own stated taxable-income and credit calculation." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model fabricated mortgage interest from the mortgage balance even though unlisted interest must be zero, and its stated itemized deductions were smaller than the standard deduction. It then submitted $5,594 despite its own erroneous itemized calculation yielding $6,722." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model explicitly derived approximately $6,116 before the $500 Credit for Other Dependents, which gives approximately $5,616, but submitted $6,104. It failed to carry its own nonrefundable-credit subtraction into the numeric output." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a projected $30,000 standard deduction instead of the 2026 $32,200 amount, overstating taxable income by $2,200. The correct $55,102 taxable income produces $6,116.24 before the correctly identified $500 dependent credit." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model derived about $5,568 after the $500 dependent credit, then inexplicably changed the result to $10,500 by invoking nonexistent self-employment adjustments and benefit phase-outs. Neither affects this federal income-tax output." +us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA rules expired for 2026, replacing the $32,200 standard deduction and 12% bracket with personal exemptions, a $16,500 deduction, and a 15% bracket. It also omitted the $500 Credit for Other Dependents." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly included $9,180 of the dependent's disability benefits in the parents' AGI and applied expired personal exemptions and pre-TCJA rates. The dependent's benefit is not part of the joint filers' $87,302 gross income, and current 2026 rules use the $32,200 standard deduction plus a $500 dependent credit." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used approximately $60,400 of taxable income instead of $55,102 and treated the household as having no qualifying dependent credit. It needed the $32,200 joint standard deduction and the $500 Credit for Other Dependents." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly applied post-expiration pre-TCJA rules, including personal exemptions and a 15% bracket. For 2026 the calculation uses the $32,200 standard deduction, the 10% and 12% brackets at this income, and the $500 Credit for Other Dependents." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model fabricated $4,500 of mortgage interest from a balance, added state taxes as itemized deductions, and applied expired personal exemptions and pre-TCJA rates. The listed facts instead produce a $32,200 standard deduction and $55,102 of taxable income." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly applied the $500 dependent credit but used a $30,800 standard deduction and outdated bracket threshold. The 2026 joint deduction is $32,200, yielding $55,102 of taxable income and $6,116.24 before the credit." +us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a $30,000 standard deduction and proxy bracket thresholds instead of the enacted 2026 values. Applying the $32,200 deduction and exact 2026 joint brackets before its correctly identified $500 credit yields $5,616.24." +us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The submitted $4,567 does not follow from the stated wages, pension, standard deduction, or dependent credit. Those inputs yield $55,102 of taxable income, $6,116.24 of regular tax, and $5,616.24 after the $500 nonrefundable credit." +us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked itemized deductions based on mortgage and medical context without computing them and submitted $4,940. The standard deduction controls because no mortgage interest is listed and deductible medical expenses do not exceed $32,200; the resulting taxable income is $55,102." +us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model wrongly added $9,180 of the dependent's disability benefits to the joint filers' taxable income. Excluding that income leaves $87,302 of gross income and $55,102 after the $32,200 standard deduction." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The unexplained $8,500 is inconsistent with the stated wage-and-pension income after the joint standard deduction and dependent credit. The correct sequence is $87,302 gross income, $55,102 taxable income, $6,116.24 regular tax, and a $500 nonrefundable credit." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied expired personal exemptions and omitted the $500 Credit for Other Dependents. It also itemized deductions rather than applying the larger $32,200 standard deduction required by the listed inputs." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed reversion to a $17,291 standard deduction and a 15% bracket, producing excessive taxable income and tax. The applicable 2026 rules retain a $32,200 joint deduction and 12% second bracket, and the disabled adult dependent qualifies for a $500 nonrefundable credit." +us,scenario_067,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model used an approximate $31,500 standard deduction and explicitly claimed no dependent, omitting the $500 Credit for Other Dependents for the disabled 23-year-old. The exact deduction is $32,200 and the credit is fully available below its phaseout threshold." +us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly identified the $500 dependent credit but used a $30,800 standard deduction and estimated bracket threshold. The exact 2026 deduction is $32,200, reducing taxable income to $55,102." +us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model wrongly included $9,180 of the dependent's disability benefits in the parents' AGI and deducted $1,845 of auto-loan interest. Neither adjustment belongs in this joint taxable-income calculation; gross income is $87,302 and taxable income is $55,102." +us,scenario_067,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model wrongly included all $10,800 of the dependent's disability benefits in joint AGI and granted a $2,000 CTC to a 23-year-old. The dependent's income is excluded from the parents' gross income, and age 23 permits only the $500 Credit for Other Dependents, even when disability removes the qualifying-child age test for dependency." +us,scenario_067,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model wrongly added the dependent's $10,800 disability benefit to the parents' gross income and used a projected $31,200 standard deduction. The joint filers have $87,302 of gross income and receive the exact $32,200 standard deduction before the correctly identified $500 dependent credit." +us,scenario_067,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model invented a $10,000 CTC and used it to eliminate the entire tentative tax. A 23-year-old dependent does not qualify for the under-17 CTC; this household receives only the $500 nonrefundable Credit for Other Dependents." +us,scenario_067,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the 23-year-old disabled dependent as a qualifying child for the refundable Additional Child Tax Credit. Disability can remove the age limit for the qualifying-child relationship test used for dependency and EITC purposes, but it does not override the CTC requirement that the child be under age 17, so the ACTC is $0." +us,scenario_067,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented five qualifying children even though the household contains only one dependent, age 23, and then constructed a $10,000 CTC and $1,210 ACTC from that nonexistent child count. The sole dependent fails the CTC under-age-17 requirement, so there is no CTC remainder to refund and no ACTC." +us,scenario_067,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly treated age 60 as above the Medicare eligibility threshold. The applicable age threshold is 65, and no disability-based Medicare pathway or other exception applies to the head, so the correct value is 0." +us,scenario_067,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly derived $6,300.54, then discarded that calculation and invented spouse wages by transferring the head's $67 hourly rate to the spouse despite the instruction that unlisted numeric inputs equal zero. Only the head's $82,360 of stated wages belongs in the payroll-tax base." +us,scenario_067,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model truncated the component taxes and then added them incorrectly: its own rounded figures of $5,106 and $1,194 total $6,300, not $6,299. Calculating without premature rounding gives $5,106.32 of Social Security tax plus $1,194.22 of Medicare tax, totaling $6,300.54." +us,scenario_067,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model named the correct taxable wages and payroll-tax components but submitted a number inconsistent with applying their statutory rates. Social Security at 6.2% and Medicare at 1.45% on $82,360 total $6,300.54, not $5,727.14." +us,scenario_067,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model replaced the exact payroll-tax calculation with an arbitrary estimate and rounded $6,300 into a broad $6,300–$6,400 range before selecting $6,376. The exact component amounts are $5,106.32 for Social Security and $1,194.22 for Medicare, totaling $6,300.54." us,scenario_067,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_067,ssi,gemini-3-flash-preview,llm_error,age_disability,False,"The model treated the dependent’s general disability flag as automatic satisfaction of SSI’s disability criterion. The dependent fails the SSI aged/blind/disabled gateway, so calculating $1,329 from an assumed federal benefit rate and unearned-income exclusion was inapplicable." -us,scenario_067,ssi,glm-5.2,llm_error,age_disability,False,"The model assumed that being age 23 and generally disabled made the dependent categorically SSI-eligible. PolicyEngine’s SSI-specific aged/blind/disabled test is false for the dependent, so neither the $20 exclusion nor the assumed federal benefit rate enters the calculation." -us,scenario_067,ssi,gpt-5.5,llm_error,age_disability,False,"The model incorrectly declared the disabled 23-year-old SSI-eligible and proceeded directly to an income-offset calculation. The dependent does not satisfy the SSI-specific aged, blind, or disabled criterion, leaving no eligible person and no payable SSI." -us,scenario_067,ssi,gpt-5.6-sol,llm_error,age_disability,False,"The model treated the dependent’s general disability status as establishing SSI disability and subtracted countable disability benefits from an estimated maximum. The SSI categorical eligibility test fails first, so the benefit is zero without applying the general income exclusion." -us,scenario_067,ssi,grok-build-0.1,llm_error,age_disability,False,"The model assumed the dependent passed SSI’s disability test and then evaluated resources, countable unearned income, and an estimated federal benefit rate. The dependent fails the SSI aged/blind/disabled gateway, making all of those downstream calculations irrelevant." -us,scenario_067,ssi,kimi-k3,llm_error,age_disability,False,"The model equated the general disability input with SSI disability eligibility and focused on the absence of parental deeming after age 18. The adult dependent does not pass PolicyEngine’s SSI-specific aged/blind/disabled test, so deeming and the $20 exclusion never affect the result." -us,scenario_067,ssi,minimax-m3,llm_error,age_disability,False,"The model incorrectly treated the dependent as categorically SSI-eligible and then used a $1,913 monthly rate rather than first applying the individual SSI eligibility gateway. No household member satisfies the SSI aged, blind, or disabled criterion, so there is no benefit from which to subtract the disability income." -us,scenario_067,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model invented a $500 disabled-dependent exemption beyond the $3,000 base exemptions and then failed to apply the exact 2.95% rate. Indiana taxes $84,302 at 2.95%, producing $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly started from federal taxable income after a federal standard deduction and invented Indiana health-premium, medical-expense, and dependent credits. Indiana instead subtracts only $3,000 of base exemptions from $87,302 and applies 2.95% to $84,302." -us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model claimed an extra $1,500 dependent exemption and selected an incorrect projected tax rate. The applicable calculation uses $3,000 of exemptions and the exact 2026 rate of 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model explicitly derived the exact computation—$87,302 minus $3,000, taxed at 2.95%, equals $2,486.91—but then submitted $2,620. Its final output contradicts its own correct arithmetic." -us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model replaced the exact 2.95% rate with an approximation of about 2.9% and rounded the taxable-income calculation loosely. Applying 2.95% to the correctly exempted income of $84,302 yields $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly deducted all $4,942 of private pension income and added a $1,500 disabled-dependent exemption, then used a 3.05% rate. The pension remains in the $87,302 AGI, only $3,000 of exemptions apply, and the rate is 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model invented $2,000 of additional dependent exemptions, used 3.05% instead of 2.95%, and then submitted an amount inconsistent with even that calculation. Indiana taxable income is $84,302 after $3,000 of exemptions." -us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model correctly derived taxable income of $84,302 but used a 3.0% flat rate. The applicable 2026 rate is 2.95%, yielding $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model added a nonexistent $1,500 disabled-dependent exemption and used 3.0% rather than 2.95%. The calculation subtracts $3,000 from $87,302 and taxes $84,302 at 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer applies an asserted 3.05% rate without showing the required $3,000 exemption calculation. Indiana's 2026 rate is 2.95% on $84,302, producing $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used the correct 2.95% rate but subtracted $4,500 rather than the applicable $3,000 of base exemptions. Taxable income is $84,302, not $82,802." -us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly calculated $84,302 of taxable income but applied 3.05%. Applying the exact 2026 rate of 2.95% gives $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly subtracted the $3,000 exemptions but used a 3.0% rate. The applicable rate is 2.95%, so tax on $84,302 is $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an incorrect 2.75% rate and failed to subtract the $3,000 of Indiana base exemptions. Indiana applies 2.95% to $84,302 rather than 2.75% to the full $87,302 AGI." -us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The unexplained $3,501 result does not follow Indiana's flat-rate calculation. Subtracting $3,000 from AGI gives $84,302, and 2.95% of that amount is $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model improperly invoked standard deductions, mortgage interest, medical expenses, and unspecified nonrefundable credits. The traced Indiana calculation subtracts only $3,000 of base exemptions from $87,302 before applying 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model applied the correct 2.95% rate but deducted $4,500 of exemptions instead of $3,000. The correct taxable income is $84,302, producing $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The model stated the correct $84,302 taxable-income base but made an arithmetic or transcription error: 3.0% of $84,302 is $2,529.06, not $2,619.06. It also used 3.0% instead of the applicable 2.95% rate." -us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used the correct 2.95% rate but invented $1,500 of additional exemptions, reducing taxable income to $82,802. Only $3,000 is subtracted, leaving $84,302." -us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $4,500 of exemptions and introduced an irrelevant exclusion for disability benefits that were not part of the filers' $87,302 AGI. The applicable exemption total is $3,000, leaving $84,302 taxable at 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The unexplained $2,200 estimate does not implement Indiana's exact exemption and flat-rate calculation. Indiana taxes $84,302 after $3,000 of exemptions at 2.95%, yielding $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly derived $84,302 of taxable income but applied 3.05% rather than 2.95%. The exact tax is $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $3,000 of Indiana base exemptions and used 2.9% instead of 2.95%. The correct base is $84,302 and the correct tax is $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required structured-output contract. -us,scenario_067,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the dependent exemption as $1,500 and added another $1,500 disabled-child exemption, for $5,000 total exemptions. Indiana subtracts $3,000, so taxable income is $84,302 at 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model inflated federal AGI to $97,202, invented a $3,800 Indiana standard deduction, and used a 3.05% rate. The calculation begins with $87,302, subtracts $3,000 of exemptions, and applies 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model first used the wrong 3.05% rate and then invented a $100 dependent credit plus $374.42 of unspecified nonrefundable credits. No such credits reduce this liability; $84,302 taxed at 2.95% equals $2,486.91." +us,scenario_067,ssi,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated the dependent’s general “is disabled” household input as automatic satisfaction of SSI’s aged/blind/disabled criterion. PolicyEngine’s SSI-specific eligibility test returned false for the dependent, so its adult-child deeming analysis, $20 exclusion, and benefit-rate subtraction never apply." +us,scenario_067,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model inferred SSI disability eligibility directly from the dependent’s general disability status. The dependent fails the SSI-specific aged/blind/disabled test, so parental asset deeming and countable unearned income are irrelevant and SSI is $0." +us,scenario_067,ssi,gpt-5.5,llm_error,categorical_eligibility,False,"The model declared the 23-year-old SSI-eligible solely because the household facts label the dependent disabled. PolicyEngine returned false for that person’s SSI aged/blind/disabled criterion, so subtracting countable disability income from an estimated federal benefit rate was an inapplicable computation." +us,scenario_067,ssi,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model skipped SSI’s categorical eligibility gate and proceeded directly to the general income exclusion and federal maximum benefit calculation. The dependent does not satisfy PolicyEngine’s SSI aged/blind/disabled criterion, so no SSI benefit calculation follows." +us,scenario_067,ssi,grok-build-0.1,llm_error,categorical_eligibility,False,"The model equated the dependent’s general disability flag with SSI disability qualification, then evaluated resources, the $20 exclusion, and an extrapolated federal benefit rate. PolicyEngine’s SSI-specific aged/blind/disabled test is false for the dependent, making all those downstream calculations inapplicable." +us,scenario_067,ssi,inkling,llm_error,categorical_eligibility,False,"The model assumed the disabled label made the dependent categorically eligible for SSI and calculated a residual benefit after the $20 exclusion. The dependent fails PolicyEngine’s SSI aged/blind/disabled test, so the correct computation ends at $0 before counting the $900 monthly disability benefit." +us,scenario_067,ssi,kimi-k3,llm_error,categorical_eligibility,False,"The model treated general disability status as sufficient for SSI eligibility and focused on the rule ending parental deeming at age 18. The dependent is not SSI aged, blind, or disabled under PolicyEngine’s program-specific test, so neither non-deeming nor the income exclusion produces a benefit." +us,scenario_067,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly declared the dependent SSI-qualified from the general disability fact and then offset disability income against a $1,913 monthly rate. PolicyEngine’s SSI categorical test is false for the dependent; additionally, $1,913 is not the applicable individual federal SSI maximum, so both the eligibility gate and downstream benefit-rate calculation were misapplied." +us,scenario_067,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model invented a $500 disabled-dependent exemption and reduced taxable income to $83,802 instead of $84,302. It then replaced the 2.95% rate with an unsupported approximate rate and unexplained minor deductions." +us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly used the federal standard deduction to determine Indiana taxable income and invented Indiana health-premium and medical-expense credits. Indiana instead subtracts $3,000 of base exemptions from $87,302 and taxes $84,302 at 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model overstated the dependent exemption, reducing taxable income to $83,802, and then selected unsupported 3.05% and 3.12% rates. The applicable computation is $84,302 multiplied by 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model explicitly derived the exact computation—$84,302 multiplied by 2.95% equals $2,486.91—but abandoned it and submitted $2,620. Its submitted value is unsupported by either calculation presented." +us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an approximate 2.9% rate instead of Indiana’s 2026 rate of 2.95% and never fixed taxable income at $84,302. Those approximations produced $2,410 instead of the direct statutory calculation." +us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented a full Indiana deduction for the head’s $4,942 private pension and an additional $1,500 disabled-dependent exemption. It also used 3.05% rather than 2.95%; the pension remains in the $87,302 starting AGI and only $3,000 of exemptions applies." +us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented $5,000 of exemptions rather than applying the $3,000 base exemptions. It also used a 3.05% rate and then increased its own calculated $2,510 to $2,650 without a supporting adjustment." +us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model correctly obtained taxable income of $84,302 but applied a 3.0% rate. Indiana’s 2026 rate is 2.95%, which produces $2,486.91." +us,scenario_067,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented an additional $1,500 disabled-dependent exemption, reducing taxable income from $84,302 to $82,802. It also applied 3.0% instead of the 2.95% rate." +us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model stated that Indiana applies 3.05% and did not identify the $3,000 exemptions or the resulting $84,302 tax base. The required computation uses 2.95% and yields $2,486.91." +us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model overstated exemptions as approximately $4,500, producing an $82,802 base. Only $3,000 of base exemptions applies, so the 2.95% rate is imposed on $84,302." +us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly calculated taxable income as $84,302 but applied 3.05%. Applying the 2026 rate of 2.95% yields $2,486.91." +us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly calculated taxable income as $84,302 but applied 3.0%. The applicable 2.95% rate produces $2,486.91." +us,scenario_067,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model applied an incorrect 2.75% rate directly to AGI and omitted the $3,000 base exemptions. Indiana taxes $84,302, not $87,302, at 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer implies an unsupported tax base or rate and supplies no computation connecting $87,302 of AGI to $3,501. Subtracting the $3,000 exemptions and applying 2.95% yields $2,486.91." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model improperly invoked standard deductions, mortgage interest, medical expenses, and unspecified nonrefundable credits. None changes this traced Indiana calculation: $87,302 less $3,000, taxed at 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model overstated Indiana exemptions as $4,500 and therefore taxed $82,802. The correct exemptions total $3,000, leaving $84,302 taxable at the same 2.95% rate." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The model’s arithmetic is internally wrong: $87,302 less $3,000, multiplied by 3.0%, equals $2,529.06, not $2,619.06. It also used 3.0% instead of the applicable 2.95% rate." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model invented another $1,500 exemption and reduced taxable income to $82,802. Indiana’s applicable exemptions total $3,000, leaving $84,302 taxable at 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied $4,500 of exemptions rather than $3,000 and gave an incoherent explanation about excluding $9,180 of disability benefits even though its stated $82,802 base starts from $87,302. The correct base is $84,302." +us,scenario_067,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model supplied only a rounded estimate with no applicable rate or exemption computation. The exact calculation is $87,302 less $3,000, multiplied by 2.95%, which yields $2,486.91." +us,scenario_067,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly obtained taxable income of $84,302 but used a 3.05% rate. Indiana’s 2026 rate is 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model taxed the full $87,302 AGI without subtracting Indiana’s $3,000 base exemptions. It also used 2.9% instead of 2.95%; the correct base is $84,302." +us,scenario_067,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted exemptions for only the two filers and omitted the dependent’s $1,000 base exemption. It also used an approximate 2.9% rate instead of 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. +us,scenario_067,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model invented a $1,500 dependent exemption and a further $1,500 disabled-child exemption, totaling $5,000 instead of $3,000. This reduced taxable income to $82,302 rather than $84,302." +us,scenario_067,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model improperly included the dependent’s disability benefits in the joint filers’ AGI and invented a $3,800 Indiana standard deduction. It also used 3.05%; the calculation instead starts from $87,302, subtracts only $3,000, and applies 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly reached $84,302 but used 3.05% and then invented a $100 dependent credit plus $374.42 of unspecified nonrefundable credits. No such adjustments enter the traced liability before refundable credits." +us,scenario_067,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model included the dependent’s $10,800 disability benefits in the joint tax unit’s AGI. The filers’ AGI is $87,302; after $3,000 of exemptions, $84,302 is taxed at 2.95%." us,scenario_067,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_068,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $67,750 of educational assistance above an IRC §127 exclusion to federal income; PolicyEngine's educational-assistance input does not enter this filer’s AGI. Its submitted $8,258 also contradicts its own stated $13,573 calculation." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model improperly used the refundable EITC to reduce an output defined before refundable credits and invented additional health-related offsets. It also annualized wages from hours despite the prompt explicitly making $34,084 the annual wage total and reversed the §127 exclusion by subtracting $5,250 from wages." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $67,750 of educational assistance to wages. The engine instead uses $34,083.53 of AGI, subtracts the $16,100 standard deduction, and taxes $17,983.53." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the educational-assistance input above $5,250 as taxable wages, inflating AGI from $34,083.53 to $101,834. That erroneous inclusion drove its taxable income and tax far above the traced amounts." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model identified the correct $17,984 taxable-income calculation and even derived approximately $1,910, then submitted $2,372 without a supporting computation. Applying the 2026 main-rate schedule to $17,983.53 yields $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $67,750 of educational assistance in AGI and used an estimated $15,750 standard deduction instead of $16,100. The engine taxes $17,983.53, not $86,084." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"After eventually choosing the correct wage-only income treatment, the model replaced its own approximately $1,915 bracket calculation with an unsupported $1,750 submission. The exact $16,100 deduction and 2026 rate thresholds produce $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model both added $67,750 of educational assistance to income and subtracted $8,389 of separately reported ESI premiums from gross wages. Neither adjustment appears in the traced AGI, which remains $34,083.53 before the $16,100 standard deduction." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset regime with a $7,600 standard deduction, personal exemption, and 15% bracket, while the engine uses the 2026 $16,100 standard deduction and current main rates. It also wrongly subtracted the separately listed ESI premium from the annual gross-wage input." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model invented a nonrefundable child or other credit despite the household containing no qualifying child and no facts supporting another credit. It also used the wrong $15,200 standard deduction and taxed all taxable income at 10%." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted $8,389 of ESI premiums from the stated annual gross wages and applied a personal exemption under an obsolete post-sunset assumption. The traced calculation starts from $34,083.53 of AGI and subtracts only the $16,100 standard deduction." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly added all $73,000 of educational assistance to income, subtracted the ESI premium, and applied a reverted tax regime with a personal exemption and 15% and 25% brackets. The engine excludes the assistance from AGI and applies the $16,100 standard deduction and 2026 main rates to wage income." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a post-TCJA-sunset $8,300 standard deduction, a $5,100 personal exemption, and a 15% second bracket. The engine instead deducts $16,100 and applies the operative 10% and 12% brackets to $17,983.53." -us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $67,750 of educational assistance in taxable wages and used a 2024 deduction and brackets as a proxy. The traced 2026 calculation excludes that input from AGI and uses the $16,100 standard deduction." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that $34,084 of wage income falls entirely below the standard deduction. After the $16,100 deduction, $17,983.53 remains taxable and generates $1,910.02 before refundable credits." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the standard deduction and unspecified nonrefundable credits as eliminating liability. The deduction leaves $17,983.53 taxable, and no nonrefundable credit reduces the resulting $1,910.02 tax." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model applied a projected post-sunset deduction-plus-personal-exemption amount and a 15% second bracket. The engine uses a $16,100 standard deduction, no personal exemption, and the operative 2026 main-rate thresholds." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $67,750 of educational assistance above the §127 limit to taxable income. PolicyEngine's traced AGI contains the $34,083.53 wage amount without that addition." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the full $73,000 educational-assistance input to wages. The engine excludes that input from AGI, leaving $17,983.53 taxable after the $16,100 standard deduction." -us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an unspecified deduction that left about $19,000 taxable instead of the traced $17,983.53. Applying the exact $16,100 deduction and 2026 thresholds yields $1,910.02 rather than $2,058." -us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly excluded educational assistance but used a 2025-like $15,000 standard deduction and 2025-like bracket thresholds. The 2026 deduction is $16,100, leaving $17,983.53 taxable and producing $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted the separately reported $8,389 ESI premium from gross wages and then applied a post-TCJA-sunset deduction and 15% bracket. The engine preserves $34,083.53 as AGI and subtracts the $16,100 standard deduction." -us,scenario_068,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_068,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model correctly excluded educational assistance but used an estimated $15,750 standard deduction and approximate rate thresholds. The exact $16,100 deduction leaves $17,983.53 taxable and yields $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model correctly kept educational assistance and ESI premiums out of the income calculation but used a $15,750 standard deduction instead of $16,100 and approximate bracket thresholds. Those parameter errors raised taxable income from $17,983.53 to $18,334 and tax from $1,910.02 to $1,957.08." -us,scenario_068,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model applied an American Rescue Plan-style expanded childless EITC schedule that is not the applicable 2026 law. Under the 2026 childless-worker EITC parameters, a single filer with $34,084 of earned income and no qualifying children is fully phased out, so the EITC component is $0." -us,scenario_068,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model treated gross wages and household size as enough to qualify for a positive EITC and did not apply the childless-worker EITC phaseout to a single adult with $34,084 of earnings. The correct computation fully phases out the childless EITC and leaves no other refundable federal credit source." -us,scenario_068,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable value or explanation for federal_refundable_credits. This is a missing-output failure rather than a substantive tax calculation. -us,scenario_068,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model invented a $20.15 Maryland employee payroll tax by treating Maryland’s progressive individual income tax as a payroll-tax component. Maryland income tax belongs outside this output, which contains only $2,113.18 of employee Social Security tax and $494.21 of Medicare tax." -us,scenario_068,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model replaced the stated $34,084 wage base with an unsupported $93,445 base, thereby treating nonwage or invented amounts as FICA wages. It also added a Maryland FAMLI employee contribution that is not part of the applicable payroll-tax calculation." -us,scenario_068,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the $8,389 employer-sponsored insurance premium from FICA wages. The prompt lists that premium as a separate household fact, not an employee pre-tax cafeteria-plan contribution, so Social Security and Medicare apply to the $34,084 wage amount." -us,scenario_068,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model assumed the listed employer-sponsored insurance premium was a pre-tax employee contribution and reduced FICA wages to $25,695. The full $34,084 of stated wages is subject to employee Social Security and Medicare tax." -us,scenario_068,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly deducted $8,389 of employer-sponsored insurance premiums from the payroll-tax base. Using the stated $34,084 wage base yields $2,113.18 of Social Security tax plus $494.21 of Medicare tax." -us,scenario_068,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model invented an additional $67,750 of payroll wages and taxed $101,834 instead of the stated $34,084 of wages. Educational assistance and other separately listed facts do not create that additional FICA wage amount." -us,scenario_068,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model named the correct wage base and tax components but submitted $2,608.414, which does not follow from its own 6.2% plus 1.45% calculation. The traced components are $2,113.18 and $494.21, totaling $2,607.39." -us,scenario_068,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model’s submitted value of $2,590.07 contradicts its own explanation, which states a total of $2,607.52. It also miscomputed the component arithmetic; the traced Social Security and Medicare amounts are $2,113.18 and $494.21, totaling $2,607.39." -us,scenario_068,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model taxed an unsupported $101,834 of compensation rather than the stated $34,084 wage amount. It further added a 0.6% Maryland FAMLI employee contribution that is not applicable in this payroll-tax output." -us,scenario_068,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model treated the $8,389 employer-sponsored insurance premium as an employee cafeteria-plan contribution excluded from FICA wages. That fact does not reduce the stated $34,084 wage base, so employee Social Security and Medicare total $2,607.39." -us,scenario_068,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required structured-output contract." -us,scenario_068,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $67,750 of educational assistance to Maryland AGI and then used a $2,700 standard deduction and reduced personal exemption. The calculation requires AGI of $34,083.53, a $3,400 standard deduction, and the full $3,200 exemption." -us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $8,389 employer-sponsored insurance premium from wages even though the listed premium is not an AGI deduction, then claimed the remaining income was eliminated by deductions. Maryland taxable income is $27,483.53 after only the $3,400 standard deduction and $3,200 personal exemption." -us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The submitted $1,029 contradicts the model's own calculations, which produced either $4,554 after adding educational assistance or $1,336 after excluding it. The required derivation excludes the educational-assistance input from AGI, subtracts $3,400 and $3,200, and taxes $27,483.53." -us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used a projected $2,700 Maryland standard-deduction cap instead of $3,400 and then introduced an unsupported adjustment after calculating the brackets. The correct taxable income is $27,483.53, and the schedule directly produces $1,252.97." -us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an obsolete approximate $2,700 standard deduction instead of $3,400 and did not execute the bracket arithmetic consistently. Subtracting $3,400 and the $3,200 exemption from $34,083.53 leaves $27,483.53." -us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $67,750 of the separately listed educational assistance as taxable Maryland income and inflated AGI to $101,834. PolicyEngine's AGI is $34,083.53, with no educational-assistance amount added, and the applicable standard deduction is $3,400." -us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly avoided adding educational assistance but used an estimated $2,550 standard deduction rather than Maryland's $3,400 amount. It also understated the resulting bracket tax; $27,483.53 of taxable income yields $1,252.97." -us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model invented Maryland AGI of $93,445 and estimated total deductions of $5,980, neither of which follows from the household inputs. The engine uses AGI of $34,083.53 and deductions totaling $6,600." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by the $8,389 employer-sponsored insurance premium to obtain AGI of $25,695 and used a $2,550 standard deduction. The premium is not subtracted in this calculation, and Maryland's standard deduction is $3,400." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $1,133 estimate understates the tax produced by Maryland's specified deductions and brackets. AGI of $34,083.53 less the $3,400 standard deduction and $3,200 exemption leaves $27,483.53, producing $1,252.97." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model improperly reduced AGI by the employer insurance premium and then added an estimated Maryland county income tax to the requested state-only output. This variable excludes local income tax; only the Maryland state schedule applies to $27,483.53." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model invented AGI of $98,695 by incorporating educational assistance into taxable income and used a $2,550 standard deduction. The correct AGI is $34,083.53 and the correct standard deduction is $3,400." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used a $2,550 standard deduction instead of $3,400, leaving taxable income $850 too high. Maryland taxable income is $27,483.53, not $28,334." -us,scenario_068,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used projected deduction and exemption amounts and then misapplied the Maryland brackets: even its stated $28,034 taxable income would yield $1,279.12, not $1,453.28. The actual $27,483.53 taxable income yields $1,252.97." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model falsely claimed that deductions and exemptions fully offset $34,084 of wages. The $3,400 standard deduction and $3,200 exemption leave $27,483.53 taxable, so liability is not zero." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly floored Maryland tax at zero without calculating taxable income or the rate schedule. Maryland deductions total $6,600, leaving $27,483.53 subject to tax." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model substituted an estimated $2,900 standard deduction for the actual $3,400 deduction. That made taxable income $500 too high and overstated the tax by $23.77." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model incorrectly raised AGI to $101,834 by treating educational assistance above $5,250 as taxable income. PolicyEngine uses AGI of $34,083.53 and does not add that educational-assistance input." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used a projected $2,850 standard deduction instead of Maryland's $3,400 deduction. The additional $550 deduction lowers taxable income to $27,483.53 and tax to $1,252.97." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly included educational assistance in Maryland income and even misstated the subtraction leading to its claimed $101,334 taxable income. The calculation starts from $34,083.53 of AGI and reaches $27,483.53 after deductions and exemption." -us,scenario_068,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model's rough $1,400 estimate did not apply the actual Maryland deduction and bracket parameters. Taxing $27,483.53 after the $3,400 deduction and $3,200 exemption gives $1,252.97." -us,scenario_068,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $2,350 standard deduction instead of $3,400, overstating taxable income by $1,050. Applying the same schedule to the correct $27,483.53 taxable income produces $1,252.97." -us,scenario_068,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $8,389 employer insurance premium to reduce AGI to $25,695 and then used a $2,400 standard deduction. The premium is not an AGI deduction here, and the Maryland standard deduction is $3,400." -us,scenario_068,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the entire $73,000 educational-assistance input as taxable because no education expenses were listed. PolicyEngine adds none of it to AGI, which remains $34,083.53, and applies a $3,400 standard deduction." -us,scenario_068,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used an indexed $2,850 standard-deduction cap instead of the applicable $3,400 deduction. This overstated taxable income by $550 and tax by $26.15." -us,scenario_068,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used an approximate $2,000 standard deduction and omitted the $3,200 personal exemption from its taxable-income calculation. Both the $3,400 standard deduction and $3,200 exemption must be subtracted, leaving $27,483.53." -us,scenario_068,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used a $3,100 standard deduction instead of $3,400, then invented a personal-exemption credit and a standard-deduction credit after already subtracting those items from income. Maryland treats them as deductions in this derivation, not additional nonrefundable credits; the resulting tax is $1,252.97." -us,scenario_068,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly identified Maryland's refundable EITC linkage but failed to carry through its own federal EITC phaseout conclusion. A single childless filer at $34,084 has no positive federal childless EITC under the applicable income limit, so Maryland's refundable percentage of federal EITC is also $0 rather than a residual $245." -us,scenario_068,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated Maryland's refundable EITC as a broad low-income worker credit and assigned a large amount without applying the childless federal EITC income phaseout. With no qualifying children and $34,084 of income, the federal EITC base is $0, so Maryland's refundable EITC percentage and any applicable refundable state-credit total are $0, not $1,500." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's reasoning correctly derived approximately $1,980 after the $192.40 Saver's Credit, but it submitted $1,697. Its numeric output contradicts its completed computation." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction instead of $16,100 and never subtracted the $192.40 Saver's Credit. Correct taxable income is $20,169.67, and tax after that nonrefundable credit is $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly computed about $2,172 of tax before credits but then changed the result to $2,398 without a valid computation. It also omitted the $192.40 Saver's Credit that reduces the requested output to $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model added the bracket components incorrectly: $1,240 plus $932 is about $2,172, not $2,549. It then incorrectly declared that no nonrefundable credit applied, omitting the $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's own bracket calculation produced about $2,076 using its mistaken deduction estimate, but it submitted $2,544 without mathematical support. It also failed to apply the $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model repeatedly substituted projected standard deductions and bracket thresholds for the applicable 2026 values. It also stated that no credits applied, omitting the $192.40 Saver's Credit on $1,924 of qualified contributions." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly derived about $2,172 before credits but supplied $1,329 without any valid subtraction connecting those figures. The applicable Saver's Credit is only $192.40, so the resulting liability is $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $600 of employee health-insurance premiums from AGI and used an incorrect standard deduction. It also counted only $1,554 of qualified retirement contributions, excluding the Roth contributions that bring the Saver's Credit base to $1,924." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA sunset regime with a $7,600 standard deduction, personal exemption, and 15% bracket. The applicable rules use a $16,100 standard deduction and 10% and 12% brackets, followed by the $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model omitted the $276 student-loan-interest deduction from its stated AGI and used a $15,500 standard deduction instead of $16,100. Its $1,494 output also does not follow from the stated income and deduction calculation or the applicable $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly subtracted a personal exemption in addition to the standard deduction and supplied a result inconsistent with the applicable 2026 rate schedule. The correct calculation uses the $16,100 standard deduction, no personal exemption, and a $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced AGI for health-insurance premiums and applied a reverted standard-deduction-plus-personal-exemption regime. Correct AGI is $36,269.67 and taxable income is $20,169.67 before applying the $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $15,700 standard deduction and the 2025 threshold of $11,925 instead of the applicable 2026 $16,100 deduction and $12,400 threshold. Its Saver's Credit amount was correct, but it applied it to an overstated tentative tax." -us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used estimated values of $15,420 for the standard deduction and $12,259 for the 10% bracket ceiling. The applicable $16,100 deduction and $12,400 ceiling produce $2,172.36 before the correctly identified $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented child, dependent-care, or other nonrefundable credits despite the household having no dependents or qualifying care facts. The only applicable nonrefundable credit is the $192.40 Saver's Credit, which does not eliminate the $2,172.36 tentative tax." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed unspecified nonrefundable credits reduced the liability to zero, contrary to the instruction that unlisted facts are false. Only the $192.40 Saver's Credit applies, leaving $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly applied projected post-TCJA-sunset rules, including a personal exemption and a 15% bracket, and stated that no nonrefundable credits applied. The applicable calculation uses the $16,100 standard deduction, 10% and 12% brackets, and the $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model correctly reached approximately $2,172.40 of tax before credits but incorrectly concluded that no nonrefundable credit was available. AGI of $36,269.67 qualifies for a 10% Saver's Credit on $1,924 of contributions, reducing tax by $192.40." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model stopped at tentative regular tax and omitted the Saver's Credit. The taxpayer's AGI and $1,924 of qualified retirement contributions generate a $192.40 nonrefundable credit." -us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The submitted $1,623 does not follow from the stated deductions: those inputs produce $20,169.67 of taxable income and $2,172.36 of tentative tax. After the $192.40 Saver's Credit, the requested amount is $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied sunset rules with an $8,300 standard deduction, a personal exemption, and a 15% bracket. The applicable 2026 rules instead provide a $16,100 standard deduction and 10% and 12% brackets; its $192 Saver's Credit treatment was otherwise close." -us,scenario_070,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted health-insurance premiums from AGI and used a $15,400 standard deduction. It also excluded the Roth retirement contributions from the Saver's Credit base, using $1,554 instead of $1,924." -us,scenario_070,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,household_unit_or_filing_status,False,"The model treated this single adult with no qualifying person as head of household and used the larger head-of-household deduction and brackets. It also excluded Roth contributions from the Saver's Credit base; the filing status is single and qualified contributions total $1,924." -us,scenario_070,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model first calculated substantial tentative tax and a limited Saver's Credit, then asserted that unspecified other nonrefundable credits eliminated the balance. No facts support such credits, and the $192.40 Saver's Credit leaves $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model omitted the $276 student-loan-interest deduction from AGI and assigned a 50% Saver's Credit rate instead of 10%. It also excluded the Roth contributions from qualified contributions and submitted $1,213.50 even though its own intermediate calculation yielded $1,475.52." -us,scenario_070,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated Illinois ACA adult expansion as a borderline gross-income test instead of applying the MAGI limit: PolicyEngine’s MAGI income is 2.27 times FPL, above the 138% FPL adult expansion threshold. It also treated employer-sponsored insurance and partial HBI-style preconditions as relevant support for eligibility, but PolicyEngine assigns no qualifying Medicaid category, so head_medicaid_eligible is 0." -us,scenario_070,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages by the $300 health insurance premium, treating it as a pre-tax exclusion from Social Security and Medicare wages. Payroll tax applies to the full $38,000 wage amount here, yielding $2,356 of Social Security tax plus $551 of Medicare tax." -us,scenario_070,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model misclassified the $300 health premium as a pre-tax employer-sponsored health deduction from FICA wages. PolicyEngine taxes the full $38,000 of wages for employee Social Security and Medicare, so the $37,700 base understates both components." -us,scenario_070,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model computed only employee Social Security tax and incorrectly set regular Medicare tax to zero because wages were below the Additional Medicare Tax threshold. The threshold only affects the 0.9% Additional Medicare Tax; the standard 1.45% Medicare tax applies to the $38,000 of wages." -us,scenario_070,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model deducted the $300 health insurance premium from wages before applying FICA, using a $37,700 payroll tax base. The payroll tax calculation uses the full $38,000 in wages for both 6.2% Social Security and 1.45% Medicare tax." -us,scenario_070,payroll_tax,minimax-m3,llm_error,other,False,"The model performed the correct component calculation, $2,356 of Social Security tax plus $551 of Medicare tax, then replaced the exact $2,907 total with an unsupported rounded value of $2,970. Payroll tax is reported as the annual dollar amount from the component sum, not rounded to a nearby larger amount." -us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model invented a $715 reduction after describing tax of $1,869 and a $120 exemption-related reduction, which would produce $1,749 rather than $1,154. Illinois instead subtracts the $2,925 exemption from AGI before applying 4.95%, yielding $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $2,850 personal exemption instead of the 2026 Illinois exemption of $2,925. Applying 4.95% to $36,269.67 minus $2,925 yields $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly approached an answer near $1,658, then discarded that calculation and submitted $1,726 based on an unexplained $34,870 taxable-income estimate. The trace-derived taxable income is $33,344.67, producing $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model computed approximately $1,655 from its stated inputs but replaced it with $1,699 based on unspecified rounding. Using the exact $36,269.67 AGI and $2,925 exemption gives $33,344.67 taxable income and $1,650.56 of tax." -us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly treated the listed health premium as a pre-tax payroll deduction and added back the traditional IRA and student-loan-interest deductions. Illinois begins here with the computed $36,269.67 AGI and subtracts the $2,925 personal exemption, rather than reconstructing $36,315 of Illinois income." -us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $276 student-loan-interest adjustment from AGI and used an estimated $2,850 exemption, then added an unsupported adjustment to its own $1,668 calculation. The applicable figures are $36,269.67 of AGI and a $2,925 exemption." -us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an unexplained AGI of $35,670 instead of $36,269.67 and an incorrect $2,842 exemption instead of $2,925. Those errors reduced its taxable-income base below the correct $33,344.67." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model invented a $135 Illinois renter's credit based on an assumed rent-to-property-tax conversion. Illinois provides no such renter credit in this calculation; tax is 4.95% of AGI after the $2,925 personal exemption." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies a taxable base of about $34,667 despite stating that a $2,775 exemption was subtracted. The correct exemption is $2,925, and the correct taxable income is $33,344.67." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a rough $2,500 personal exemption rather than the 2026 amount of $2,925. Subtracting the correct exemption from $36,269.67 and applying 4.95% yields $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced AGI by an unsupported additional $300 and used a $2,775 exemption. The correct inputs are $36,269.67 of AGI and a $2,925 exemption, leaving $33,344.67 taxable." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $2,850 personal exemption instead of Illinois's 2026 exemption of $2,925. This overstated taxable income by $75 and tax by $3.73." -us,scenario_070,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model omitted the student-loan-interest deduction and used an outdated $2,425 exemption, producing $1,689 by its own arithmetic, then submitted an unexplained $1,626. The correct AGI and exemption produce $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model incorrectly concluded that ordinary deductions eliminate Illinois liability even though $33,344.67 remains taxable after the personal exemption. Applying Illinois's 4.95% rate produces $1,650.56, not zero." -us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model treated the retirement and student-loan adjustments as sufficient to reduce state tax to zero. They leave AGI of $36,269.67 and taxable income of $33,344.67 after the Illinois exemption." -us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an invented $1,824 personal exemption and omitted the student-loan-interest adjustment from its AGI. Illinois uses a $2,925 exemption against AGI of $36,269.67." -us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a $2,850 personal exemption instead of the 2026 Illinois value of $2,925. The correct taxable income is $33,344.67, taxed at 4.95%." -us,scenario_070,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The submitted $1,777 implies approximately $35,899 of taxable income, so the model failed to carry through the trace-derived AGI and full personal exemption. Illinois taxable income is $33,344.67 after subtracting $2,925 from $36,269.67." -us,scenario_070,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated AGI by $299.67 and used an outdated $2,425 personal exemption. The exact calculation uses $36,269.67 minus $2,925, not $35,970 minus $2,425." -us,scenario_070,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model applied 4.95% directly to AGI because it incorrectly asserted that Illinois has no personal exemption. Illinois subtracts the 2026 personal exemption of $2,925 before applying the rate." -us,scenario_070,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used a $2,775 personal exemption rather than the 2026 amount of $2,925. This overstated taxable income by about $150 and produced $1,658 instead of $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied a standard deduction to Illinois income and reduced the tax base to about $20,696. Illinois uses AGI minus the $2,925 personal exemption here, leaving $33,344.67 taxable." -us,scenario_070,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model treated the $2,800 personal exemption as a dollar-for-dollar nonrefundable credit after already using it to reduce taxable income. Illinois's personal exemption is a $2,925 subtraction from AGI, not a credit against tax." -us,scenario_070,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model invented a small Illinois EITC after correctly recognizing that the childless federal EITC is fully phased out at this income. Illinois' refundable EITC is tied to the federal earned income credit calculation and does not create a positive credit for a single 57-year-old with about $38,100 of income and no qualifying children when the federal EITC is zero." -us,scenario_071,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a New York aged Medicaid pathway and treated the head's age plus Social Security income as enough to qualify, despite PolicyEngine assigning medicaid_category = NONE. It also misread the resource condition by saying $62,291 was below a typical senior Medicaid asset limit even while citing a much lower $15,900-style threshold, and it never applied the trace fact that no SSI or other qualifying category exists." -us,scenario_071,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model forced the person into an aged Medicaid category even though the engine assigns no Medicaid category. Its conclusion depends on unspecified senior disregards and Social Security exclusions reducing $36,542 of annual income below an aged limit, but the reference trace has no SSI-related or aged pathway qualifying this person." -us,scenario_071,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly noticed that $36,542 exceeds the 100% and 138% FPL-style thresholds, then overrode that result by treating Medicare eligibility and Medicare Savings Program concepts as full Medicaid eligibility. PolicyEngine does not assign a Medicare Savings, aged, or expansion category here; medicaid_category is NONE and MAGI is 2.28 FPL." -us,scenario_071,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated New York medically needy spenddown and excess-income concepts as a modeled full Medicaid eligibility route for this case. PolicyEngine's trace shows no qualifying aged, disabled, SSI-related, MAGI, or spenddown category, and the listed $240 of medical expenses does not convert this 2.28 FPL case into Medicaid eligibility." -us,scenario_071,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 76 as creating an elderly Medicaid pathway subject only to a vague income/resource screen. In the engine trace, age and Medicare-style status do not establish a Medicaid category, and the person is assigned medicaid_category = NONE." -us,scenario_071,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model used a generic elderly-with-limited-income shortcut instead of applying PolicyEngine's categorical eligibility logic. The correct derivation assigns no Medicaid category, so the head is not Medicaid eligible despite being 76 and having retirement income." -us,scenario_071,head_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model applied the New York MAGI Medicaid threshold to a 76-year-old and said income was below it. PolicyEngine's trace gives MAGI income as 2.28 times FPL, which is above the MAGI category limit, and no non-MAGI category applies." -us,scenario_071,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly computed that $36,542 minus $240 of medical expenses still exceeds the cited aged income limit, then incorrectly converted excess income into eligibility through spenddown. PolicyEngine assigns no Medicaid category for this person, and the small listed medical expenses do not establish a modeled spenddown pathway to full Medicaid eligibility." -us,scenario_071,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented wage income from weekly hours by assuming a New York minimum-wage-equivalent hourly rate, directly violating the instruction that unlisted numeric inputs are 0. With no listed wages or self-employment earnings, the listed Social Security retirement income and private pension are not subject to employee payroll tax, so applying Social Security and Medicare rates to imputed wages produced the erroneous $2,500." -us,scenario_071,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model counted the New York household credit as a refundable state credit. That credit is nonrefundable and therefore does not enter state_refundable_credits; its $63 estimate is exactly the wrong credit bucket, not a qualifying refundable New York credit." -us,scenario_072,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model treated Medicare eligibility as requiring an explicitly provided Medicare status instead of applying the age-based Medicare eligibility rule. The head is age 80, so PolicyEngine marks the head Medicare-eligible regardless of income, assets, or stated current coverage." -us,scenario_072,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model incorrectly treated regular Medicare payroll tax as applying only above a threshold. The threshold applies to Additional Medicare Tax, while the 1.45% employee Medicare tax applies to all $20,000 of wages, adding $290 to the $1,240 Social Security tax." -us,scenario_072,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model computed the correct FICA components, $1,240 Social Security plus $290 Medicare, and then replaced the $1,530 total with an unsupported rounded estimate of $1,460. Payroll tax is an annual dollar amount from exact statutory rates here, so no rounding step changes $1,530 to $1,460." -us,scenario_072,payroll_tax,minimax-m3,llm_error,other,False,"The model's reasoning correctly derives $1,530 from $1,240 Social Security and $290 Medicare with no Additional Medicare Tax or Michigan employee payroll tax, but it submitted $1,550. This is a final-answer transcription or arithmetic error inconsistent with its own stated calculation." -us,scenario_072,spouse_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model treated Medicare eligibility as unavailable unless an explicit Medicare eligibility or enrollment status was provided. It missed the standard age-based Medicare pathway: the spouse is 77, above the age-65 eligibility threshold, so spouse_medicare_eligible is 1." -us,scenario_072,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used only personal exemptions and a senior interest-and-dividend subtraction, then taxed the remaining wages. It failed to apply Michigan's Tier 2 joint standard deduction against all income for a qualifying taxpayer age 67 or older, which reduces taxable income to zero." -us,scenario_072,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,The answer applies Michigan's flat rate after subtracting Social Security and ordinary personal exemptions. It omits the age-based Tier 2 joint standard deduction that eliminates the remaining Michigan income. -us,scenario_072,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,The model reduced the Tier 2 standard deduction by the household's Social Security benefits and then discarded it in favor of personal exemptions. Michigan's qualifying age-67 standard-deduction pathway applies against the household's remaining income and leaves zero taxable income. -us,scenario_072,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model subtracted only two estimated personal exemptions from $20,166 and also used an incorrect 4.05% rate. The decisive error is omitting Michigan's Tier 2 joint standard deduction for the elderly couple, which eliminates the taxable-income base before any rate is applied." -us,scenario_072,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model correctly excluded Social Security and veterans benefits but stopped after applying ordinary personal exemptions. It failed to use the Tier 2 age-67 joint standard deduction against all remaining income, so it incorrectly left a positive taxable balance." -us,scenario_072,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model treated the couple's age relief as enhanced personal exemptions and taxed the residual $7,566. Michigan's applicable Tier 2 standard deduction instead offsets all remaining Michigan income, producing no taxable income." -us,scenario_072,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model removed taxable Social Security from federal AGI but then allowed only two $5,600 personal exemptions. It omitted the larger Tier 2 joint standard deduction available through the age-67 pathway, which reduces Michigan taxable income to zero." -us,scenario_072,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used estimated $5,000 personal exemptions and an incorrect 4.05% rate after subtracting taxable Social Security. It never applied Michigan's Tier 2 joint standard deduction for taxpayers age 67 or older, which wipes out the remaining income." -us,scenario_072,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model substituted two estimated $6,100 personal exemptions for Michigan's applicable elderly-taxpayer deduction pathway. The Tier 2 joint standard deduction applies against the remaining income and leaves a zero tax base." -us,scenario_072,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model cycled among speculative personal and senior exemption amounts and ultimately taxed income remaining after two $6,500 exemptions. It failed to apply the Tier 2 joint standard deduction against all income for the age-67-or-older household, which eliminates the entire Michigan tax base." -us,scenario_073,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated receipt of SSDI as proof that the head qualifies through an aged/disabled Medicaid pathway. SSDI is not SSI and does not by itself establish that pathway; the head receives no SSI, qualifies through no Medicaid category, and the asset discussion cannot create categorical eligibility." -us,scenario_073,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the desired traditional 401(k) contribution from Medicaid MAGI despite the absence of wages or other earnings from which that employee contribution could be made. The engine derives MAGI at 1.60 times FPL, above Michigan's expansion limit, so the adult expansion pathway fails." -us,scenario_073,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model converted SSDI receipt and a generic characterization of income as low into Medicaid eligibility without identifying a valid Michigan pathway. The head's MAGI is 1.60 times FPL and SSDI receipt alone does not establish an aged/disabled category, leaving the Medicaid category as NONE." -us,scenario_073,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that the head meets Michigan's expansion-group income condition, but the engine-derived MAGI is 1.60 times FPL. That exceeds the expansion threshold, and no alternative Medicaid category applies." -us,scenario_073,head_medicaid_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model excluded all SSDI from Medicaid MAGI merely because it characterized the benefits as nontaxable, reducing countable income to the $3,108 pension. The engine's MAGI calculation is 1.60 times FPL, so the head exceeds the expansion threshold and qualifies through no other category." -us,scenario_073,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model expressly invented completion of the 24-month SSDI waiting period from the full-year constant-status instruction. That instruction establishes receipt throughout 2026, not SSDI entitlement for the preceding 24 months, so the 57-year-old does not qualify for Medicare." -us,scenario_073,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model converted an unspecified duration of SSDI receipt into an established disability status with a completed 24-month waiting period. Neither prior-year receipt nor another under-65 Medicare qualification is listed, so SSDI income does not produce eligibility." -us,scenario_073,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model correctly identified the 24-month SSDI rule but treated that waiting period as completed without any supporting fact. Full-year SSDI income in 2026 does not establish 24 preceding months of entitlement, leaving the under-65 head ineligible." -us,scenario_073,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model treated receipt of Social Security disability benefits as conferring Medicare eligibility immediately. Under-65 SSDI recipients must complete the applicable 24-month entitlement waiting period, which is not listed here." -us,scenario_073,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model inferred that a full-year, ongoing SSDI benefit had already satisfied Medicare’s 24-month waiting period. Constant status throughout the tax year does not backdate SSDI entitlement into earlier years, so the disability pathway is not established." -us,scenario_073,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"The model inserted completion of the required Medicare waiting period despite no fact giving the SSDI entitlement start date. At age 57, current-year SSDI receipt without a completed 24-month period does not establish Medicare eligibility." -us,scenario_073,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model treated SSDI recipient status alone as sufficient for Medicare. An under-65 recipient must satisfy the 24-month entitlement waiting period or another listed qualifying condition, neither of which is present." -us,scenario_073,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model collapsed the SSDI and Medicare eligibility rules into an immediate qualification. The head is under 65, and the facts do not establish completion of Medicare’s 24-month SSDI waiting period." -us,scenario_073,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,The model treated the income label “Social Security disability income” as proof of Medicare-eligible disability status. That label supplies neither a Medicare eligibility flag nor the 24 months of SSDI entitlement required for the under-65 pathway. -us,scenario_073,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,The model inferred that full-year SSDI income automatically activates Medicare’s disability pathway. Full-year receipt only describes 2026 and does not establish that the 24-month SSDI waiting period was completed before or during the year. -us,scenario_073,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model treated SSDI income as establishing disability-based Medicare eligibility by itself. For a 57-year-old, eligibility requires the completed SSDI waiting period or another qualifying condition, and neither is listed." -us,scenario_073,head_medicare_eligible,gpt-5.6-terra,llm_error,age_disability,False,The model incorrectly stated that receipt of Social Security disability income directly confers Medicare eligibility. It omitted the 24-month SSDI entitlement waiting period applicable to the under-65 disability pathway. -us,scenario_073,head_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model attributed an automatic SSDI-to-Medicare rule to PolicyEngine. The supplied SSDI amount does not establish completion of the 24-month waiting period, and the head is only 57 with no other Medicare-qualifying condition." -us,scenario_073,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's stated reasoning correctly identifies that no W-2 wages are provided and that Social Security disability and pension income are not employee payroll-tax wages, which yields payroll_tax = 0. Its submitted value of 2846 directly violates its own explanation and the prompt's requirement that the numeric value match the explanation." -us,scenario_073,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated failure of the ordinary net-income test and a negative formula allotment as requiring a $0 benefit. It omitted Michigan TANF non-cash categorical eligibility and the resulting minimum allotment, which totals $287.68 over the year." -us,scenario_073,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income limit and a $2,750 asset limit as disqualifying tests. Michigan's TANF non-cash categorical-eligibility pathway applies here, and the $3,600 of assets passes the applicable asset test, producing the annual minimum allotment of $287.68." -us,scenario_073,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model deducted roughly $5,165 of medical costs as though it were a monthly expense, driving monthly net income below zero. The trace instead yields monthly net income of $1,755.47 and a negative regular allotment, followed by the categorical minimum totaling $287.68 annually." -us,scenario_073,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model stopped after concluding that net income exceeded the ordinary limit and that the formula benefit was exhausted. It omitted TANF non-cash categorical eligibility, under which the eligible one-person household receives the minimum allotment totaling $287.68." -us,scenario_073,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model used about $587 as the maximum monthly allotment for a one-person household; the applicable maximum is $298. It also overstated deductible medical premiums, while the trace yields $1,755.47 of monthly net income and therefore the categorical minimum totaling $287.68." -us,scenario_073,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model treated the 100% FPL net-income test as dispositive after calculating approximately $1,752 per month. It failed to apply Michigan TANF non-cash categorical eligibility and the minimum-allotment rule, which produces $287.68 annually." -us,scenario_073,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model equated a fully offset regular allotment with no SNAP payment. Because the household is categorically eligible through TANF non-cash eligibility, the negative regular formula is replaced by the minimum allotment totaling $287.68." -us,scenario_073,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,The model used the ordinary 130% FPL gross-income test as an automatic disqualifier. The household qualifies through Michigan's TANF non-cash categorical-eligibility pathway and receives the annual minimum allotment of $287.68. -us,scenario_073,snap,gemini-3-flash-preview,llm_error,period_annualization,False,"The model identified categorical eligibility and the minimum benefit but rounded it to a constant $23 for all 12 months. The applicable monthly minimum is mostly $23.84 and sometimes $24.37 as the poverty guideline adjusts, totaling $287.68." -us,scenario_073,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The answer treats countable income above the ordinary threshold as ending eligibility. Michigan TANF non-cash categorical eligibility preserves eligibility and triggers the minimum allotment, yielding $287.68 annually." -us,scenario_073,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model concluded that the lack of shelter deductions and a fully offset regular allotment require a zero benefit. It omitted the minimum allotment for this categorically eligible household, which totals $287.68 for the year." -us,scenario_073,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model correctly recognized the disability-related gross-test treatment but stopped when 30% of its net-income figure exceeded the maximum allotment. TANF non-cash categorical eligibility converts that nonpositive formula result into the minimum allotment totaling $287.68. -us,scenario_073,snap,gemini-3.6-flash,llm_error,period_annualization,False,"The model found the correct categorical-eligibility and minimum-allotment pathway but used a flat $23 monthly amount. PolicyEngine applies monthly minimums of mostly $23.84 and occasionally $24.37, producing $287.68 rather than $276." -us,scenario_073,snap,glm-5.2,llm_error,thresholds_rates,False,"The model calculated 30% of $1,053.08 as $315.92; the correct product is $315.92 only after ordinary multiplication, but it then used that negative regular result to assign $0 despite categorical eligibility. The applicable minimum-allotment rule replaces the nonpositive result and totals $287.68 annually." -us,scenario_073,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The answer applies ordinary one-person income limits as an automatic bar. Michigan TANF non-cash categorical eligibility makes the household eligible for the minimum allotment, totaling $287.68." -us,scenario_073,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required additional earnings, housing-cost, or take-up facts to establish SNAP. The prompt sets unlisted amounts to zero and directs program take-up, while TANF non-cash categorical eligibility produces the $287.68 annual minimum allotment." -us,scenario_073,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model ignored that receipt of Social Security disability income establishes the relevant disabled status and then applied the ordinary gross-income test as dispositive. The household also qualifies through Michigan TANF non-cash categorical eligibility and receives $287.68 annually. -us,scenario_073,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model treated a nonpositive ordinary allotment after deductions and the 30% contribution as a zero payment. Categorical eligibility requires the minimum allotment instead, totaling $287.68 over the year." -us,scenario_073,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model recognized disability and deductions but concluded that income fully eliminating the regular formula benefit produces $0. It omitted the minimum allotment attached to Michigan TANF non-cash categorical eligibility, totaling $287.68." -us,scenario_073,snap,grok-4.3,llm_error,categorical_eligibility,False,The model treated income above the ordinary one-person net limit as conclusive ineligibility. Michigan TANF non-cash categorical eligibility supplies an independent eligibility pathway and results in the $287.68 annual minimum allotment. -us,scenario_073,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model correctly identified the disabled-household treatment but equated a fully offset regular allotment with $0. It failed to apply categorical eligibility's minimum allotment, which totals $287.68 annually." -us,scenario_073,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model stopped after its medical deduction left 30% of net income above the maximum allotment. For this TANF non-cash categorically eligible household, that nonpositive regular result is replaced by the minimum allotment totaling $287.68." -us,scenario_073,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model concluded that a 30% contribution exceeding the maximum allotment eliminates SNAP entirely. It omitted Michigan TANF non-cash categorical eligibility and the corresponding minimum benefit of $287.68 annually. -us,scenario_073,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model subtracted only the standard deduction and omitted the allowed excess medical deduction, overstating monthly net income at about $1,942 instead of $1,755.47. It also failed to apply the categorical minimum after the regular formula became nonpositive, yielding $287.68 rather than $0." -us,scenario_073,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly states that the household exceeds both income and asset limits, while it counts only $1,800 of the listed $3,600 in bank and stock assets. The full $3,600 passes the applicable asset test, and TANF non-cash categorical eligibility leads to the $287.68 annual minimum allotment." -us,scenario_073,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,The model refused to treat listed SSDI as establishing disability and applied the ordinary 130% FPL gross-income test as dispositive. The household qualifies through Michigan TANF non-cash categorical eligibility and receives the minimum allotment totaling $287.68. +us,scenario_068,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It incorrectly added $67,750 of educational assistance to taxable income even though this PolicyEngine input does not enter AGI. Its submitted $8,258 also contradicts its own stated $13,573 calculation instead of applying the rates to $17,983.53 of taxable income." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It improperly used the refundable EITC to reduce a quantity defined before refundable credits and invented additional offsets from employer health premiums. It also annualized wages from hours despite the prompt explicitly making $34,084 the full-year wage total." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It incorrectly treated $67,750 of the educational-assistance input as taxable wages, inflating AGI from $34,083.53 to $101,834. The correct taxable income after the $16,100 standard deduction is $17,983.53." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It incorrectly included educational assistance above a $5,250 Section 127 exclusion in AGI. PolicyEngine does not include this educational-assistance input in federal AGI, leaving only employment income before the standard deduction." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Its reasoning reaches the correct $17,983.53 taxable-income base and approximately $1,910 of bracket tax, but it submitted $2,372 without a supporting computation. Applying the 2026 main rates to that base yields $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It incorrectly added $67,750 of educational assistance to wages and also used an estimated $15,750 standard deduction instead of $16,100. Those choices inflated taxable income from $17,983.53 to $86,084." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"After eventually selecting the correct wages-only income treatment, it abandoned its own bracket calculation near $1,915 and submitted an unsupported $1,750. The exact $16,100 deduction and 2026 brackets produce $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It both added $67,750 of educational assistance to taxable income and subtracted the separately listed $8,389 employer-sponsored insurance premium from gross wages. Neither adjustment belongs in this AGI calculation, and it also used an estimated deduction rather than $16,100." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It incorrectly applied a post-TCJA-sunset regime with a $7,600 standard deduction, personal exemption, and 15% bracket. It also deducted $8,389 of employer-sponsored insurance premiums from the stated annual gross wages; the applicable calculation uses a $16,100 standard deduction and current 10%/12% rates." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It invented a nonrefundable child or other credit for a single filer with no dependents or qualifying credit facts. It also used a $15,200 deduction and taxed all taxable income at 10%, whereas the $16,100 deduction and progressive 2026 rates yield $1,910.02 with no credit reduction." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It wrongly subtracted the $8,389 employer-sponsored insurance premium from gross wages and applied an obsolete sunset regime combining a standard deduction with a personal exemption. The stated wages remain AGI and receive the $16,100 standard deduction." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It incorrectly added all $73,000 of educational assistance to income, deducted the insurance premium, and applied a reverted-tax-law standard deduction, personal exemption, and rates. PolicyEngine instead taxes $17,983.53 after the $16,100 standard deduction under the 2026 main rates." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It applied an expired-law $8,300 standard deduction, a $5,100 personal exemption, and a 15% marginal bracket. The applicable $16,100 standard deduction leaves $17,983.53 taxable under the 2026 10% and 12% brackets." +us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It incorrectly included $67,750 of educational assistance in taxable wages and used the 2024 standard deduction and brackets as a proxy. The educational-assistance input does not enter AGI, and the 2026 deduction is $16,100." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated $34,084 of wages as fully sheltered by the standard deduction. The $16,100 deduction leaves $17,983.53 of positive taxable income, producing $1,910.02 before refundable credits." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It incorrectly concluded that the standard deduction eliminated the wage income and floored the tax at zero. The deduction leaves $17,983.53 taxable, so no zero floor applies." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied a projected post-sunset deduction-plus-personal-exemption regime and a 15% marginal rate. The applicable $16,100 standard deduction and 2026 10%/12% brackets yield $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It incorrectly treated $67,750 of educational assistance above a $5,250 exclusion as taxable income. PolicyEngine excludes this input from AGI, leaving employment income of $34,083.53 before the standard deduction." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It incorrectly added the entire $73,000 educational-assistance amount to wages. That input does not enter federal AGI in this calculation, so taxable income is $17,983.53 rather than $90,984." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used an unspecified deduction that left about $19,000 taxable instead of applying the exact $16,100 standard deduction. The resulting taxable income is $17,983.53, and the exact 2026 brackets produce $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It correctly excluded educational assistance but used the 2025 $15,000 standard deduction and 2025-like bracket thresholds. Replacing those proxies with the 2026 $16,100 deduction and applicable thresholds reduces the tax to $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It incorrectly subtracted $8,389 of employer-sponsored insurance premiums from the stated gross wages, then applied a post-TCJA-sunset $8,527 deduction and 15% bracket. The premiums do not reduce this wage input, and the applicable standard deduction is $16,100." +us,scenario_068,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It supplied no numeric output or explanation for the requested variable, so the required structured answer was missing." +us,scenario_068,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It correctly kept educational assistance out of AGI but used an estimated $15,750 standard deduction and approximate bracket thresholds. The exact $16,100 deduction applied to $34,083.53 of employment income yields $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It correctly excluded educational assistance but used a $15,750 standard deduction instead of $16,100 and approximate bracket thresholds. The exact inputs leave $17,983.53 taxable and produce $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It improperly subtracted the refundable EITC when computing federal income tax before refundable credits and also claimed the standard deduction reduced taxable income to zero. The $16,100 deduction leaves $17,983.53 taxable, and EITC is excluded from this output." +us,scenario_068,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model incorrectly projected the temporary American Rescue Plan expansion of the childless EITC into 2026 and therefore used a phaseout endpoint near $33,000–$34,000. Under 2026 law, the temporary expansion is not in effect, and $34,084 exceeds the applicable childless EITC income limit, yielding zero EITC." +us,scenario_068,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model treated wages of $34,084 and a one-person household as qualifying for EITC without applying the childless single-filer phaseout endpoint. At that income the childless EITC is fully phased out, and no other refundable credit applies." +us,scenario_068,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, violating the required output contract." +us,scenario_068,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model assigned a $4,000 EITC despite the taxpayer being single, childless, and earning $34,084. The childless EITC is fully phased out at this income, and the absence of qualifying children also leaves the refundable CTC at zero." +us,scenario_068,head_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly treated $34,084 as below Maryland's Medicaid income threshold. The applicable comparison uses MAGI income at 2.14 times FPL, which exceeds the adult MAGI limit, and the head has no non-MAGI eligibility category." +us,scenario_068,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly added $20.15 of Maryland income tax to the employee payroll-tax output. Maryland individual income tax belongs outside this output, and the applicable payroll tax consists only of the traced Social Security and Medicare amounts totaling $2,607.39." +us,scenario_068,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model replaced the listed $34,084 of wages with an unsupported $93,445 FICA base, thereby treating non-wage household information as payroll compensation. It also added a Maryland FAMLI employee contribution that does not apply in the reference-year calculation." +us,scenario_068,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model deducted the $8,389 employer-sponsored insurance premium from FICA wages as though the prompt established a pre-tax Section 125 salary-reduction contribution. The listed premium does not reduce the traced payroll-tax base, so Social Security and Medicare apply to the engine's full wage amount." +us,scenario_068,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated the ESI premium as a confirmed pre-tax payroll deduction and reduced FICA wages from $34,084 to $25,695. The prompt supplies an insurance-premium amount but no Section 125 exclusion, so the engine does not subtract it from the payroll-tax base." +us,scenario_068,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly removed $8,389 of ESI premiums from Social Security and Medicare wages. Applying FICA to the traced wage base yields $2,607.39, not the rounded $1,966 produced from the reduced $25,695 base." +us,scenario_068,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model invented an additional $67,750 of payroll wages and taxed a $101,834 base even though the only listed wages are $34,084. Educational assistance is not an additional wage input and cannot be converted into the unexplained compensation amount used here." +us,scenario_068,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model identified the correct federal components and wage amount but performed their arithmetic incorrectly: 6.2% plus 1.45% on the traced wage base produces the engine components $2,113.18 and $494.21, totaling $2,607.39. Its submitted $2,608.414 does not follow from its stated calculation." +us,scenario_068,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's submitted value contradicts its own explanation: it explained a $2,607.52 total but returned $2,590.07. It also misstated the component arithmetic instead of using the traced $2,113.18 Social Security and $494.21 Medicare liabilities." +us,scenario_068,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model taxed an invented $101,834 wage base rather than the listed $34,084 of wages, improperly incorporating non-wage information into payroll compensation. It then added an inapplicable 0.6% Maryland FAMLI contribution." +us,scenario_068,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model assumed the ESI premium was paid through a pre-tax cafeteria plan and excluded it from FICA wages, a fact the prompt does not provide. The engine therefore does not reduce the payroll base to $25,695 and instead computes $2,607.39 from its traced wage base." +us,scenario_068,payroll_tax,inkling,llm_error,state_local_rule,False,"The model correctly included employee Social Security and Medicare but incorrectly added a 0.45% Maryland FAMLI contribution. No such mandatory Maryland employee payroll tax applies to this 2026 reference calculation, leaving only the two federal components totaling $2,607.39." +us,scenario_068,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so it failed the required structured-output contract." +us,scenario_068,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model identified the correct Social Security and Medicare components but rounded their sum to $2,600 instead of adding the liabilities at monetary precision. The traced components are $2,113.18 and $494.21, which total $2,607.39." +us,scenario_068,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance to wages even though the applicable AGI is $34,083.53. It also used a $2,700 standard deduction and reduced the personal exemption instead of subtracting $3,400 and $3,200." +us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It treated the $8,389 employer-sponsored insurance premium as a direct subtraction from the stated wages and then asserted that ordinary deductions eliminated the remaining income. Maryland taxable income is $27,483.53 after the $3,400 standard deduction and $3,200 exemption, not zero." +us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its submitted $1,029 does not follow either computation in its own reasoning, which produced approximately $4,554 under one treatment and $1,336 under the other. The required derivation uses $34,083.53 of AGI and deductions of $3,400 and $3,200, producing $1,252.97." +us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used an estimated $2,700 standard-deduction cap instead of the applicable $3,400 deduction, overstating taxable income by $700. It then introduced an unsupported post-bracket adjustment from about $1,286 to $1,308." +us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used an approximate $2,700 standard deduction instead of $3,400 and did not execute the Maryland brackets consistently. The correct taxable income is $27,483.53 and the rate schedule yields $1,252.97." +us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It incorrectly added $67,750 of educational assistance to the $34,084 wage amount, inflating AGI to $101,834. The applicable AGI is $34,083.53, from which the $3,400 standard deduction and $3,200 exemption are subtracted." +us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It correctly avoided adding educational assistance but used an estimated $2,550 standard deduction rather than $3,400. It also misapplied the brackets, since even its stated $28,334 taxable base would not produce $1,050." +us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It invented Maryland AGI of $93,445 and estimated combined deductions of $5,980. The traced amounts are $34,083.53 of AGI, a $3,400 standard deduction, and a $3,200 exemption." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $25,695 by subtracting the employer-sponsored insurance premium directly from wages, then used only a $2,550 standard deduction. The applicable AGI remains $34,083.53 and the standard deduction is $3,400." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its unexplained estimate understates the tax generated by Maryland's schedule on the correct $27,483.53 taxable income. Subtracting the $3,400 standard deduction and $3,200 exemption from $34,083.53 and applying the brackets yields $1,252.97." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It both reduced AGI incorrectly to $25,695 and added an estimated county income tax to the requested state-only output. County tax is excluded from state_income_tax_before_refundable_credits, while the state computation uses $34,083.53 of AGI." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used an invented AGI of $98,695, effectively treating educational assistance as taxable additional income. The calculation instead starts from $34,083.53 and subtracts $3,400 and $3,200." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used a $2,550 standard deduction instead of $3,400, leaving taxable income $850 too high. The correct taxable base is $27,483.53, producing $1,252.97." +us,scenario_068,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It substituted projected deduction and exemption amounts and then misapplied the Maryland rate schedule: its $1,453.28 does not follow from its stated $28,034 taxable income. The applicable deductions are $3,400 and $3,200, and tax on $27,483.53 is $1,252.97." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that deductions and exemptions fully offset $34,084 of wages. They total $6,600, leaving $27,483.53 subject to Maryland income tax." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It incorrectly floored the liability at zero after unspecified deductions and credits. The $3,400 standard deduction and $3,200 exemption leave $27,483.53 taxable, and no nonrefundable credit reduces the resulting $1,252.97 to zero." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It estimated the Maryland standard deduction at $2,900 rather than using $3,400. That inflated taxable income from $27,483.53 to about $27,984 and consequently overstated the tax." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance to wages and used AGI of $101,834. The applicable AGI is $34,083.53, so the tax must be computed on $27,483.53 after the two Maryland deductions." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used a projected $2,850 standard deduction instead of the applicable $3,400. The additional $550 deduction lowers taxable income to $27,483.53 and tax to $1,252.97." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It treated educational assistance as additional taxable income, producing a taxable base above $100,000 instead of $27,483.53. It also used a $2,550 standard deduction rather than $3,400." +us,scenario_068,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its rough $1,400 estimate did not apply the specified Maryland deductions and graduated rates exactly. Tax on the correct $27,483.53 taxable base is $1,252.97." +us,scenario_068,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used a $2,350 standard deduction instead of $3,400, overstating taxable income by $1,050. Applying the schedule to $27,483.53 rather than $28,534 yields $1,252.97." +us,scenario_068,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $25,695 by treating employer-sponsored insurance premiums as a direct subtraction from wages and then used a $2,400 standard deduction. The traced AGI is $34,083.53 and the standard deduction is $3,400." +us,scenario_068,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It approximated the standard deduction as $2,650 and the personal exemption as $3,400 instead of using $3,400 and $3,200. Those errors left its taxable-income estimate $550 above the correct $27,483.53." +us,scenario_068,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 educational-assistance amount to wages because no education expenses were listed. That amount does not enter the applicable AGI, which is $34,083.53, and Maryland tax is computed after deductions of $3,400 and $3,200." +us,scenario_068,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used a projected $2,850 standard-deduction cap rather than the applicable $3,400 deduction. This overstated taxable income by $550 and produced $1,279.12 instead of $1,252.97." +us,scenario_068,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It allowed only an estimated $2,000 standard deduction and omitted the $3,200 personal exemption from its taxable-income arithmetic. The two applicable subtractions total $6,600 and leave $27,483.53 taxable." +us,scenario_068,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used a $3,100 standard deduction rather than $3,400, then incorrectly converted the personal exemption and standard deduction into additional tax credits after already subtracting them from income. These deductions reduce taxable income once; they are not separate nonrefundable credits." +us,scenario_068,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It treated EITC as eliminating Maryland taxable income and the pre-refundable-credit liability. EITC does not reduce taxable income in this computation, and the $3,400 deduction plus $3,200 exemption leave $27,483.53 taxable with $1,252.97 due before refundable credits." +us,scenario_068,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly recognized that the childless federal EITC phases out to zero at this income but then invented a $245 residual Maryland refundable credit. Applying Maryland’s percentage to a federal EITC of zero yields $0, not a minimal positive amount." +us,scenario_068,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated $34,084 as qualifying low income without applying the federal childless EITC phaseout ceiling, then assigned a Maryland EITC amount exceeding the credit pathway it cited. With no qualifying children, federal EITC is zero at this income, so the percentage-based Maryland refundable EITC and the total state refundable credits are both $0." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's reasoning correctly derived approximately $2,172 before credits and a $192 Saver's Credit, which yields approximately $1,980. It then submitted $1,697, a value that contradicts every computation in its explanation." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction instead of the 2026 single-filer amount of $16,100 and never applied the $192.40 Saver's Credit. Correct taxable income is $20,169.67, and $2,172.36 of pre-credit tax falls to $1,979.96 after that credit." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly computed taxable income near $20,170 and tax near $2,172, then replaced that result with $2,398 without a supporting calculation. It also failed to subtract the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The stated bracket arithmetic is wrong: $1,240 plus $932 equals about $2,172, not $2,549. The model also incorrectly declared that no nonrefundable credit applied, omitting the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an incorrect $16,900 standard deduction and then submitted $2,544 despite its own bracket calculation producing about $2,076. It also omitted the $192.40 Saver's Credit; the applicable $16,100 deduction produces $2,172.36 before that credit." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an incorrect $15,700 standard deduction rather than $16,100 and explicitly concluded that no nonrefundable credits applied. The qualified retirement contributions generate a $192.40 Saver's Credit, reducing $2,172.36 to $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly reached approximately $2,172 before credits but then subtracted an unexplained $843 to submit $1,329. The actual Saver's Credit is $192.40 on $1,924 of qualified contributions, yielding $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $600 of employee health-insurance premiums from AGI and used an incorrect standard deduction. It also counted only $1,554 of retirement contributions for the Saver's Credit, excluding the qualified Roth contributions; the credit base is $1,924 and the credit is $192.40." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA sunset regime with a $7,600 standard deduction, personal exemption, and 15% bracket. The applicable 2026 rules use a $16,100 single standard deduction and 10%/12% brackets, followed by the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model omitted the $276 student-loan-interest deduction from its stated AGI calculation and used a $15,500 standard deduction rather than $16,100. Its $1,494 result also does not follow from the resulting taxable income or the applicable $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model subtracted a personal exemption that does not belong in this 2026 computation and produced $2,576 without reconciling that deduction with the applicable brackets. The correct calculation uses the $16,100 standard deduction alone and subtracts a $192.40 Saver's Credit from $2,172.36." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced AGI by health-insurance premiums and applied an obsolete reverted-law combination of standard deduction and personal exemption. AGI is $36,269.67, taxable income is $20,169.67, and the correctly computed $192.40 Saver's Credit leaves $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $15,700 standard deduction and the wrong 10% bracket endpoint, overstating taxable income and pre-credit tax. With the $16,100 deduction and 2026 brackets, pre-credit tax is $2,172.36; subtracting its correctly identified $192.40 Saver's Credit yields $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an incorrect $15,420 standard deduction and an incorrect $12,259 bracket threshold. Its Saver's Credit calculation was correct, but applying that credit to the correct $2,172.36 pre-credit tax yields $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented child, dependent-care, or other nonrefundable credits even though the household contains no child or dependent and lists no qualifying care expenses. Only the $192.40 Saver's Credit applies, leaving $1,979.96 rather than zero." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed unspecified nonrefundable credits eliminated the tax despite the instruction that unlisted facts are false or zero. The listed facts support only a $192.40 Saver's Credit against $2,172.36 of tax, leaving $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly applied post-TCJA-sunset parameters, including a personal exemption and a 15% bracket, and stated that no nonrefundable credit applied. The 2026 calculation uses the $16,100 standard deduction and 10%/12% brackets, then subtracts the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model correctly derived taxable income near $20,170 and tax of $2,172.40 but incorrectly concluded that no nonrefundable credit was available. AGI of $36,269.67 qualifies for a 10% Saver's Credit on $1,924 of contributions, reducing tax by $192.40." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model stopped at the regular tax produced by the 10% and 12% brackets and omitted the Saver's Credit. The taxpayer's AGI and $1,924 of qualified retirement contributions generate a $192.40 nonrefundable credit, leaving $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model submitted $1,623 without showing bracket arithmetic or identifying a credit that could reduce the correctly derived $2,172.36 to that amount. The only applicable nonrefundable credit is the $192.40 Saver's Credit, producing $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied sunset-law parameters: an $8,300 standard deduction, a $5,300 personal exemption, and a 15% bracket. The applicable 2026 calculation instead uses a $16,100 standard deduction and 10%/12% brackets; after its correctly identified $192.40 Saver's Credit, liability is $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted health-insurance premiums from AGI, used an incorrect $15,400 standard deduction, and limited the Saver's Credit base to traditional contributions. Health premiums do not reduce this AGI, and the qualified traditional and Roth contributions total $1,924, producing a $192.40 credit." +us,scenario_070,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,household_unit_or_filing_status,False,"The model treated the single adult with no dependent as a head-of-household filer and used the larger head-of-household standard deduction and brackets. The filing status is single, and it also excluded qualified Roth contributions from the Saver's Credit base; the correct credit is $192.40 on $1,924." +us,scenario_070,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model stated that pre-credit tax was about $2,108 and the Saver's Credit about $159, then claimed those credits reduced liability to zero. A credit of that size cannot eliminate the liability; the exact computation is $2,172.36 minus $192.40, or $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model omitted the $276 student-loan-interest deduction from AGI and assigned a 50% Saver's Credit rate instead of the applicable 10% rate at AGI of $36,269.67. It also excluded the Roth contributions from the credit base and submitted $1,213.50 despite its own intermediate result of $1,475.52." +us,scenario_070,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a Credit for Other Dependents even though the household contains only the taxpayer and no dependent. The only applicable nonrefundable credit is the $192.40 Saver's Credit, which reduces $2,172.36 to $1,979.96 rather than zero." +us,scenario_070,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model applied the Additional Child Tax Credit earned-income formula despite the household having no qualifying child, so there was no Child Tax Credit balance available to refund. Its separate EITC rationale also misstates the rule: $100 of investment income does not exceed the limit; the childless EITC is zero because $38,000 of earnings exceeds its phaseout ceiling." +us,scenario_070,head_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model called eligibility borderline without calculating the stated household’s MAGI-to-FPL ratio. At 2.27 times FPL, the head exceeds Illinois’s 138% FPL ACA expansion limit, and age 57, low assets, and employer-sponsored insurance do not establish another Medicaid category." +us,scenario_070,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed $300 health insurance premium as a pre-tax FICA exclusion even though no pre-tax treatment was specified. FICA applies to the full $38,000, yielding $2,356 of Social Security tax and $551 of Medicare tax." +us,scenario_070,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $300 employer-sponsored health premium from FICA wages by assuming it was employee-paid through a pre-tax plan. With no such status listed, both Social Security and Medicare taxes use the full $38,000 wage base." +us,scenario_070,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model conflated regular Medicare tax with the Additional Medicare Tax and set all Medicare tax to zero because wages were below the additional-tax threshold. The ordinary 1.45% Medicare tax applies from the first dollar of covered wages, adding $551 to the correctly computed $2,356 Social Security tax." +us,scenario_070,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages from $38,000 to $37,700 by assigning unlisted pre-tax treatment to the $300 health premium. Applying 6.2% Social Security and 1.45% Medicare rates to the full wages produces $2,907." +us,scenario_070,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly derived $2,356 of Social Security tax plus $551 of Medicare tax, then replaced the exact $2,907 total with $2,970 under an invalid claim of rounding. Rounding $2,907 does not produce $2,970, and the requested annual amount is the exact derived total." +us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $2,425 exemption instead of $2,925 and incorrectly characterized the exemption as a tax credit. It then submitted $1,154 despite its own intermediate calculations supporting neither that amount nor the correct $1,650.56 liability." +us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $2,850 personal exemption instead of the 2026 Illinois amount of $2,925. Subtracting $2,925 from $36,269.67 yields $33,344.67, whose 4.95% tax is $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model cycled through incorrect exemption and taxable-income estimates, then submitted $1,726 even though none of its displayed calculations produced that answer. The required calculation uses the $2,925 exemption and $33,344.67 of taxable income." +us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an approximate $2,850 exemption and then arbitrarily increased the resulting estimate from about $1,655 to $1,699. The exact $2,925 exemption produces $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly added back the traditional IRA and student-loan-interest deductions and assumed the listed health premium reduced wages as a pre-tax payroll deduction. It also used a stale $2,425 exemption; Illinois instead taxes $36,269.67 minus the $2,925 exemption." +us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $276 student-loan-interest deduction from adjusted gross income and used an estimated $2,850 exemption instead of $2,925. It then increased its own computed $1,668 figure to $1,690 without a valid adjustment." +us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an unsupported AGI of $35,670 and an incorrect $2,842 exemption. The trace yields AGI of $36,269.67, a $2,925 exemption, and taxable income of $33,344.67." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model invented a $135 Illinois renter's credit based on 25% of rent and subtracted it from the liability. Rent does not generate that Illinois nonrefundable property-tax credit, and the applicable $2,925 personal exemption leaves tax of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used a $2,775 personal exemption instead of $2,925 and its submitted $1,716 does not follow from the stated 4.95% calculation. The correct taxable base is $33,344.67." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model approximated the personal exemption at $2,500 rather than applying the 2026 amount of $2,925. That overstated taxable income and the Illinois liability." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated adjusted gross income at $35,970 by treating the listed health premium as an AGI reduction, then used a $2,775 exemption instead of $2,925. The correct inputs are $36,269.67 of AGI and a $2,925 exemption." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's AGI was essentially correct, but it used a $2,850 exemption instead of $2,925. The additional $75 exemption reduction lowers tax by $3.71 to $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the $276 student-loan-interest deduction and used a stale $2,425 exemption. It also submitted $1,626 even though its own stated base and rate produced about $1,689, creating an unsupported final value." +us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model incorrectly concluded that retirement deductions and the personal exemption eliminated Illinois taxable income. They leave $33,344.67 taxable at 4.95%, producing $1,650.56 before refundable credits." +us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model treated the listed adjustments as sufficient to reduce Illinois liability to zero. After those adjustments, AGI remains $36,269.67 and the $2,925 exemption leaves $33,344.67 subject to the 4.95% tax." +us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an unsupported $1,824 personal exemption rather than the 2026 Illinois exemption of $2,925. It also omitted the student-loan-interest deduction when deriving AGI, overstating both the base and tax." +us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a $2,850 personal exemption instead of $2,925. Applying the correct exemption to $36,269.67 of AGI produces $33,344.67 of taxable income and $1,650.56 of tax." +us,scenario_070,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The submitted $1,777 implies a taxable base of about $35,899, so the model failed to apply the full $2,925 personal exemption to the $36,269.67 AGI. The correct taxable base is $33,344.67." +us,scenario_070,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated AGI at $35,970 and used a stale $2,425 exemption. Its stated figures also produce about $1,660.48 rather than exactly $1,660; the traced base is $36,269.67 minus $2,925." +us,scenario_070,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used an approximate $2,775 personal exemption instead of the 2026 amount of $2,925. This overstated taxable income by $150 and tax by about $7.43." +us,scenario_070,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model incorrectly asserted that Illinois provides no personal exemption and taxed the entire $36,270 AGI. Illinois subtracts the $2,925 personal exemption before applying the 4.95% rate." +us,scenario_070,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used a $2,775 personal exemption instead of $2,925. The correct exemption reduces taxable income to $33,344.67 and tax to $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied a standard deduction to Illinois income, reducing the base to about $20,696. Illinois uses the traced AGI less the $2,925 personal exemption, not a federal-style standard deduction." +us,scenario_070,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model treated the $2,800 personal exemption as both an income subtraction and a dollar-for-dollar nonrefundable credit, then zeroed out the tax. The Illinois personal exemption reduces taxable income by $2,925; it is not a $2,925 tax credit." +us,scenario_070,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The submitted $1,658.66 implies that the model used an incorrect personal-exemption amount when subtracting from federal AGI. Using the $2,925 exemption yields taxable income of $33,344.67 and tax of $1,650.56." +us,scenario_070,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly found that the childless federal EITC was fully phased out but then invented a $102 Illinois EITC despite Illinois calculating its credit as a percentage of the federal amount. The age expansion changes categorical eligibility only; it does not override the federal income phaseout, so 20% of a zero federal EITC yields $0." +us,scenario_071,head_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model declared $36,542 of income below New York’s aged Medicaid limits and simultaneously treated $62,291 as below a resource limit it described as $15,900, reversing both comparisons. It therefore invented an aged Medicaid pathway even though the person’s engine category is NONE." +us,scenario_071,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model invoked unspecified senior disregards to reduce $36,542 below an aged Medicaid limit of roughly $20,000 without calculating any applicable disregard. No aged pathway is established, and the engine assigns medicaid_category = NONE." +us,scenario_071,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly found income above both 100% and 138% FPL thresholds, then incorrectly converted Medicare eligibility and Medicare Savings Program pathways into full Medicaid eligibility. Being Medicare-eligible does not itself confer Medicaid eligibility, and this person qualifies through no Medicaid category." +us,scenario_071,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model substituted Medicaid Buy-In, asset-test elimination, and excess-income spend-down pathways without establishing the disability, work, spend-down amount, or other conditions those pathways require. The stated $36,542 income remains far above its own cited aged limit, and the engine assigns no Medicaid category." +us,scenario_071,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 76 as sufficient to enter an elderly Medicaid pathway and skipped the required category and financial tests. The person qualifies for no Medicaid category, so age alone yields no eligibility." +us,scenario_071,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed that being elderly with “limited” income and assets establishes Medicaid eligibility without applying any pathway-specific test. The engine assigns medicaid_category = NONE, while the person’s MAGI income is 2.28 times FPL and SSI receipt is zero." +us,scenario_071,head_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model stated that the person’s income is below New York’s MAGI Medicaid threshold, but the traced MAGI income is 2.28 times FPL. It also failed to account for the absence of any qualifying Medicaid category." +us,scenario_071,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated $240 of medical expenses as enough to establish spend-down eligibility even though subtracting them leaves $36,302, far above its cited $21,024 annual limit. The existence of a spend-down program and some medical expenses does not satisfy the spend-down, and the person qualifies through no Medicaid category." +us,scenario_071,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented wage income by multiplying the listed 20 weekly work hours by an assumed New York minimum wage, despite the instruction that unlisted numeric inputs are zero. Social Security and Medicare employee taxes therefore apply to a zero wage base, not the model's imputed earnings, so payroll_tax is $0." +us,scenario_071,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly classified an estimated New York household credit as refundable. That credit is nonrefundable, so its asserted $63 belongs outside state_refundable_credits; the household qualifies for no refundable New York credit, yielding $0." +us,scenario_072,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model treated Medicare eligibility as an unlisted boolean that defaults to false instead of deriving it from age. The head is 80, exceeds the age-65 threshold, and is Medicare-eligible; enrollment status need not be provided." +us,scenario_072,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model incorrectly treated the Additional Medicare Tax threshold as an exemption from regular Medicare tax. Regular employee Medicare tax applies to all $20,000 of wages, adding $290 to the $1,240 Social Security tax for a total of $1,530." +us,scenario_072,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly calculated $1,240 of Social Security tax plus $290 of Medicare tax and explicitly obtained $1,530, then replaced that exact result with an unsupported rounded value of $1,460. Payroll tax is not rounded from $1,530 to $1,460." +us,scenario_072,payroll_tax,minimax-m3,llm_error,other,False,"The model's stated components correctly total $1,530, but it submitted $1,550. This is an arithmetic/output inconsistency: $1,240 plus $290 equals $1,530, with no Additional Medicare Tax or Michigan employee payroll tax to supply the extra $20." +us,scenario_072,spouse_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model treated the absence of an explicit Medicare-status input as disqualifying and ignored the age-based eligibility rule. At age 77, the spouse exceeds the standard Medicare eligibility age of 65, so spouse_medicare_eligible equals Yes." +us,scenario_072,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used personal exemptions and a senior interest-and-dividend subtraction instead of the joint $40,000 Michigan standard deduction available for taxpayers born from 1946 through 1952. That standard deduction eliminates the approximately $20,166 remaining after Social Security is excluded, leaving no taxable income." +us,scenario_072,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer applies Michigan’s rate after only personal exemptions and the Social Security subtraction. It omits the joint $40,000 Michigan standard deduction for the spouses’ birth-year cohort, which reduces the remaining income to zero." +us,scenario_072,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the joint $40,000 Michigan standard deduction by the household’s Social Security benefits and then discarded it in favor of personal exemptions. Social Security is separately subtracted, while the elected standard deduction offsets the approximately $20,166 of remaining income in full." +us,scenario_072,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model deducted only two estimated personal exemptions from Michigan income. It missed the alternative joint $40,000 Michigan standard deduction for taxpayers born from 1946 through 1952, which wipes out the remaining taxable base." +us,scenario_072,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model correctly excluded Social Security and veterans benefits but then used personal exemptions rather than the spouses’ joint $40,000 Michigan standard deduction. The larger deduction eliminates all approximately $20,166 of remaining Michigan income." +us,scenario_072,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model treated the spouses as receiving only senior personal exemptions and taxed the resulting $7,566 base. Their birth-year cohort instead permits a joint $40,000 Michigan standard deduction, which reduces the remaining income to zero." +us,scenario_072,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted taxable Social Security and two ordinary personal exemptions but omitted the joint $40,000 Michigan standard deduction. Electing that deduction offsets all of the approximately $20,166 left in Michigan income." +us,scenario_072,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used $10,000 of personal exemptions after removing taxable Social Security. It missed the alternative joint $40,000 Michigan standard deduction for the spouses’ birth years, which eliminates the entire remaining income base." +us,scenario_072,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used approximately $12,000 of personal exemptions rather than electing the joint $40,000 Michigan standard deduction available to this birth-year cohort. The standard deduction offsets the approximately $20,166 remaining after the Social Security subtraction." +us,scenario_072,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model applied two $6,100 personal exemptions and taxed the residual $7,966. It omitted the alternative joint $40,000 Michigan standard deduction, which reduces the household’s remaining Michigan income to zero." +us,scenario_072,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model repeatedly varied estimated senior personal-exemption amounts and settled on $13,000 rather than applying the applicable joint $40,000 Michigan standard deduction. That deduction, together with the separate Social Security exclusion, eliminates the entire remaining Michigan income base." +us,scenario_073,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated receipt of SSDI as automatically establishing disability-based Medicaid eligibility and assumed the reported assets satisfied that pathway. At age 57, with no SSI and no established qualifying category, the head does not enter an aged or disabled pathway; the engine assigns Medicaid category NONE." +us,scenario_073,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the desired traditional 401(k) contribution from Medicaid MAGI, even though the household has no wages from which that elective deferral could occur. The resulting MAGI is 1.60 times FPL, not below the 138% expansion limit, and nontaxable Social Security benefits are added back for Medicaid MAGI." +us,scenario_073,head_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled the SSDI and pension income as low without applying the expansion threshold. Those amounts produce MAGI of 1.60 times FPL, above Michigan’s 138% FPL expansion limit, and no alternative Medicaid category applies." +us,scenario_073,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model asserted that the head met the expansion-group income condition without computing the applicable income ratio. The head’s MAGI equals 1.60 times FPL and therefore exceeds the 138% FPL expansion threshold. +us,scenario_073,head_medicaid_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model excluded the entire $22,646 of SSDI merely because it was not taxable. Medicaid MAGI adds nontaxable Social Security benefits back into income, producing MAGI of 1.60 times FPL and disqualifying the head from Michigan’s expansion group." +us,scenario_073,head_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model equated having no earnings with having sufficiently low Medicaid income and ignored the SSDI and taxable pension. Those income sources yield Medicaid MAGI of 1.60 times FPL, above the expansion limit, while the head qualifies through no other category." +us,scenario_073,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model invented completion of the 24-month SSDI waiting period from the instruction that listed facts remain constant during the tax year. Full-year receipt in 2026 does not establish 24 prior months of entitlement, so the 57-year-old does not satisfy the disability-based Medicare pathway." +us,scenario_073,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated an “established disability status” as proof that the 24-month SSDI waiting period had elapsed. The prompt provides no entitlement duration, and unlisted qualifying statuses are false, leaving the under-65 head ineligible." +us,scenario_073,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,The model converted receipt of SSDI into automatic under-65 Medicare eligibility without establishing the required waiting period. An SSDI income amount alone does not show 24 months of entitlement or another qualifying condition. +us,scenario_073,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model incorrectly stated that Social Security disability benefits confer Medicare eligibility regardless of age. For an under-65 recipient, disability-based Medicare eligibility requires additional qualifying conditions that were not listed." +us,scenario_073,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model inferred that full-year, ongoing SSDI receipt meant the 24-month waiting period was satisfied. Constancy throughout one tax year establishes neither prior entitlement duration nor a separate Medicare-qualifying condition." +us,scenario_073,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"The model assumed completion of the SSDI waiting period even though no duration of entitlement was supplied. With the head under 65 and unlisted qualifying statuses set to false, the disability pathway is not established." +us,scenario_073,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model treated SSDI receipt itself as sufficient for Medicare eligibility. It omitted the under-65 pathway's additional qualification requirements, including the applicable entitlement waiting period." +us,scenario_073,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,The model equated Social Security disability income with Medicare eligibility. The income entry does not establish completion of the required SSDI entitlement period or another under-65 Medicare condition. +us,scenario_073,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model treated reported Social Security disability income as evidence of a Medicare-eligible disability status. The prompt requires unlisted statuses to be false, and it supplies neither the requisite entitlement duration nor another qualifying condition." +us,scenario_073,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model assumed that full-year SSDI income automatically activates the disability-based Medicare pathway. A full-year income amount does not establish the required period of SSDI entitlement, so the age-57 head remains ineligible." +us,scenario_073,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model treated SSDI income as establishing disability-based Medicare eligibility by itself. Under age 65, the head also needs the applicable qualifying entitlement history or another listed Medicare condition, neither of which is present." +us,scenario_073,head_medicare_eligible,gpt-5.6-terra,llm_error,age_disability,False,The model incorrectly asserted that receipt of Social Security disability income directly confers Medicare eligibility. It skipped the separate under-65 qualification requirements and inferred a status the prompt did not provide. +us,scenario_073,head_medicare_eligible,inkling,llm_error,age_disability,False,The model invented satisfaction of the 24-month SSDI waiting period and also treated a premium category that explicitly excludes Medicare Part B as evidence of Medicare coverage. Neither fact establishes an under-65 Medicare-qualifying condition. +us,scenario_073,head_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model incorrectly read PolicyEngine's rules as making SSDI income alone sufficient for Medicare eligibility. The head is under 65, and no qualifying entitlement duration or other Medicare condition was supplied." +us,scenario_073,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that the household has no payroll-taxable wages and explicitly derived payroll tax of $0, but then submitted $2,846. Its numeric output contradicts its own payroll-tax-base reasoning and final stated derivation." +us,scenario_073,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the ordinary 100%-FPL net-income test and negative formula result as disqualifying. It omitted Michigan’s TANF non-cash categorical-eligibility pathway, under which the eligible one-person household receives the minimum allotment." +us,scenario_073,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary 130%-FPL gross-income limit and a $2,750 resource limit instead of Michigan’s categorical-eligibility rules. The household passes the applicable gross-income and $3,600 asset tests and qualifies through TANF non-cash categorical eligibility." +us,scenario_073,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model converted annual medical expenses into a monthly deduction, subtracting roughly $5,165 from one month’s income, and therefore drove net income to zero. It also used a full $298 monthly maximum instead of the minimum allotment produced by the actual $1,755.47 monthly net income." +us,scenario_073,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,The model made the ordinary net-income limit dispositive after recognizing the disability pathway. It omitted TANF non-cash categorical eligibility and therefore failed to apply the one-person minimum allotment when the formulaic benefit was negative. +us,scenario_073,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model double-counted overlapping premium fields as medical deductions and used an erroneous one-person maximum allotment of about $587. The trace yields $1,755.47 monthly net income, a $298 maximum, and therefore the minimum allotment rather than $214 per month." +us,scenario_073,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model treated failure of the ordinary 100%-FPL net-income test as ending eligibility. Michigan’s TANF non-cash categorical-eligibility pathway keeps the household eligible and triggers the minimum allotment. +us,scenario_073,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model concluded that a negative result from maximum allotment minus 30% of net income means no benefit. For this categorically eligible one-person household, that result is floored at the SNAP minimum allotment." +us,scenario_073,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied only the ordinary 130%-FPL gross-income test. It omitted Michigan’s higher gross-income pathway and TANF non-cash categorical eligibility, which establish eligibility and lead to the minimum allotment." +us,scenario_073,snap,gemini-3-flash-preview,llm_error,period_annualization,False,"The model correctly identified categorical eligibility and the minimum-allotment rule but hard-coded $23 for all 12 months. PolicyEngine applies monthly minimums of mostly $23.84 and some $24.37 as the poverty guideline changes, totaling $287.68." +us,scenario_073,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The answer implies that the model applied an ordinary income cutoff as dispositive. It omitted TANF non-cash categorical eligibility, which makes the household eligible for the one-person minimum allotment." +us,scenario_073,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model treated the absence of shelter deductions and a negative formulaic allotment as producing zero. Categorical eligibility instead floors the benefit at the one-person minimum allotment. +us,scenario_073,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model correctly found that 30% of net income exceeds the maximum allotment but incorrectly converted that result to zero. The categorically eligible one-person household receives the minimum allotment. +us,scenario_073,snap,gemini-3.6-flash,llm_error,period_annualization,False,"The model correctly applied categorical eligibility and the minimum benefit but assumed a flat $23 monthly amount. The applicable monthly minimum is mostly $23.84 and sometimes $24.37, producing $287.68 rather than $276 annually." +us,scenario_073,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model concluded that a negative formulaic allotment is paid as zero even after finding the household income-eligible. For a categorically eligible one-person household, the benefit is floored at the minimum allotment." +us,scenario_073,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The answer applies a generic one-person income limit and stops. It omits Michigan’s TANF non-cash categorical-eligibility pathway, which yields the minimum allotment." +us,scenario_073,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model wrongly required earned income, housing costs, or an explicit take-up indicator to establish SNAP. The prompt assumes take-up, and the listed income and assets establish TANF non-cash categorical eligibility and the minimum allotment." +us,scenario_073,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model ignored the listed SSDI as disability status and applied the ordinary gross-income test. The household also qualifies through Michigan’s TANF non-cash categorical-eligibility pathway, producing a minimum allotment." +us,scenario_073,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,The model treated income high enough to erase the formulaic allotment as producing no SNAP. Categorical eligibility floors the negative formula result at the one-person minimum allotment. +us,scenario_073,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model correctly recognized disability and deductions but stopped when counted income erased the ordinary allotment. It omitted the minimum benefit payable through categorical eligibility. +us,scenario_073,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model treated the ordinary net-income limit as the sole eligibility rule. It omitted Michigan’s TANF non-cash categorical eligibility, under which the household receives the minimum allotment." +us,scenario_073,snap,grok-4.5,llm_error,categorical_eligibility,False,The model correctly determined that the 30% contribution fully offsets the maximum allotment but incorrectly set the benefit to zero. Categorical eligibility requires the one-person minimum allotment. +us,scenario_073,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model treated a negative maximum-minus-contribution calculation as zero. For this TANF non-cash categorically eligible household, the negative result is replaced by the minimum allotment." +us,scenario_073,snap,inkling,llm_error,period_annualization,False,"The model correctly identified the minimum-allotment rule but rounded it to $23 for every month. The engine uses mostly $23.84 and some $24.37 monthly amounts, totaling $287.68." +us,scenario_073,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model concluded that a contribution exceeding the maximum allotment eliminates SNAP. It omitted the minimum allotment retained by a categorically eligible one-person household. +us,scenario_073,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model deducted only the standard deduction and omitted the applicable medical-expense deduction, overstating monthly net income as about $1,942 instead of $1,755.47. It also failed to apply the categorical-eligibility minimum after the expected contribution exceeded the maximum allotment." +us,scenario_073,snap,minimax-m3,llm_error,asset_resource,False,"The model misread the assets as $1,800 and asserted failure of both income and resource tests. Combined bank and stock assets are $3,600 and pass the applicable asset test, while TANF non-cash categorical eligibility yields the minimum allotment." +us,scenario_073,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,The model refused to treat receipt of SSDI as disability for SNAP and applied only the ordinary 130%-FPL gross-income limit. It omitted Michigan’s TANF non-cash categorical-eligibility pathway and the resulting minimum allotment. +us,scenario_073,snap,qwen3.8-max,llm_error,asset_resource,False,"The model treated $3,600 of bank and stock assets as exceeding the applicable resource limit. The household passes the asset test and qualifies through TANF non-cash categorical eligibility, so it receives the minimum allotment." us,scenario_074,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_074,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_074,head_medicaid_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model understated Medicaid MAGI by counting only the IRA distribution and a taxable portion of Social Security while omitting income included by the engine's MAGI computation. The engine calculates MAGI at 2.13 times FPL, so the head exceeds Louisiana's expansion-adult limit and qualifies through no other category." -us,scenario_074,head_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model constructed a $20,230 MAGI by including only half of SSDI, the IRA distribution, and the state tax refund, then compared that amount with 138% FPL. The engine's complete MAGI computation produces 2.13 times FPL, above the expansion-adult limit, and no alternative Medicaid category applies." -us,scenario_074,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated receipt of SSDI as establishing disability-based Medicaid eligibility and also incorrectly described Social Security benefits as largely excluded from Medicaid MAGI. SSDI is not an automatic Medicaid pathway; with SSI equal to zero and MAGI at 2.13 times FPL, the head qualifies through neither a disability-related category nor expansion Medicaid." -us,scenario_074,head_medicaid_eligible,gpt-5.4-mini,llm_error,health_coverage,False,"The model labeled the reported sources as producing very low countable income without performing the full Medicaid MAGI computation. That computation yields 2.13 times FPL, above Louisiana's expansion-adult limit, and the head qualifies through no other pathway." -us,scenario_074,head_medicaid_eligible,gpt-5.6-terra,llm_error,health_coverage,False,"The model assumed the listed taxable income was the complete Medicaid MAGI base and therefore placed the head under the expansion-adult limit. The engine's MAGI calculation yields 2.13 times FPL, so the expansion pathway fails and no other eligibility category applies." -us,scenario_074,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted year-long SSDI receipt into completion of Medicare’s 24-month disability-entitlement waiting period. Constant status for one year does not supply the missing prior entitlement duration, so the 54-year-old fails the applicable Medicare eligibility test." -us,scenario_074,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model asserted that already receiving SSDI proves the 24-month Medicare waiting period was completed. SSDI receipt does not encode entitlement duration, and the prompt requires the unlisted duration to be zero; at age 54, the head is not eligible." -us,scenario_074,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated constant year-long SSDI status as satisfaction of the 24-month waiting period. One full tax year is only 12 months, and no earlier SSDI entitlement was listed, leaving the 54-year-old outside Medicare eligibility." -us,scenario_074,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model inferred prior SSDI entitlement from constant disability income during the tax year and thereby invented completion of the 24-month waiting period. The prompt forbids that inference, and the head is below age 65." -us,scenario_074,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model treated SSDI income itself as conferring immediate Medicare eligibility under PolicyEngine’s disability rule. Disability-based eligibility requires the relevant entitlement-duration condition; with that unlisted and set to zero, the age-54 head is not eligible." -us,scenario_074,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model explicitly assumed an ongoing pre-year SSDI history despite the instruction not to infer unlisted facts. Constant receipt during 2026 does not prove 24 months of entitlement, so the under-65 disability pathway is not satisfied." -us,scenario_074,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"The model equated SSDI receipt with Medicare eligibility and omitted the 24-month disability-entitlement waiting period. With no qualifying duration supplied and age only 54, the eligibility result is 0." -us,scenario_074,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model treated Social Security disability income as sufficient for Medicare eligibility. It failed to apply the disability-entitlement duration requirement, while the head also fails the age-65 test." -us,scenario_074,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model used SSDI income as an automatic Medicare qualifier. Receipt alone does not establish completion of the 24-month waiting period, and the head is 54 rather than at least 65." -us,scenario_074,head_medicare_eligible,gemini-3.6-flash,llm_error,age_disability,False,"The model incorrectly made SSDI receipt dispositive of Medicare eligibility. The missing entitlement history is set to zero under the prompt, so neither the disability-duration condition nor the age threshold is met." -us,scenario_074,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model misstated PolicyEngine’s rule as treating every SSDI recipient as Medicare eligible regardless of age. SSDI income does not substitute for the required disability-entitlement duration, and this 54-year-old does not meet the age pathway." -us,scenario_074,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model acknowledged failure of the age test but then activated a disability pathway solely from SSDI income. The prompt supplies no completed 24-month entitlement period and assigns zero to that unlisted input, so the disability pathway also fails." -us,scenario_074,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated SSDI receipt during the year as sufficient disability-based Medicare eligibility. It omitted the 24-month entitlement-duration requirement, leaving the age-54 head ineligible." -us,scenario_074,head_medicare_eligible,grok-4.3,llm_error,age_disability,False,"The model claimed that full-year SSDI receipt meets a 24-month waiting period. A full tax year provides only 12 months, and the prompt supplies no prior entitlement, so the waiting period is not met." -us,scenario_074,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,"The model cited the statutory waiting period but treated current SSDI receipt as proof that it had elapsed. No entitlement start date or prior duration was listed, so the under-65 Medicare pathway is not established." -us,scenario_074,head_medicare_eligible,grok-build-0.1,llm_error,age_disability,False,"The model expressly assumed that ongoing SSDI receipt meant the 24-month waiting period was complete. That assumption violates the instruction to set unlisted facts to zero; at age 54, the head therefore has no qualifying pathway." -us,scenario_074,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model treated SSDI income as automatically qualifying under the disability pathway. It omitted the required entitlement-duration condition, and the head is below the age-65 threshold." -us,scenario_074,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model applied a payroll-tax calculation to nonwage income even while stating the income was Social Security disability. Employee Social Security and Medicare taxes apply to wages, and the household has no listed wage or salary income, so the correct employee-side payroll-tax base is zero." -us,scenario_074,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable payroll_tax output or supporting explanation. The requested output was required, and the correct derivation gives zero because no wage, salary, self-employment, or mandatory state payroll-tax base is listed." -us,scenario_075,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived approximately $14,777 using the $16,100 standard deduction, enacted brackets, and 15% preferential rate, but then submitted $12,523 without any supporting computation. Its final value contradicts every calculation in its explanation." -us,scenario_075,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated a 62-year-old as qualifying for an age-65 additional standard deduction and then invented a $5,252 disability-related tax reduction. Disability at age 62 creates neither that additional standard deduction nor the claimed direct reduction, so the applicable deduction is $16,100 and no such credit applies." -us,scenario_075,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used estimated bracket cutoffs of $12,150 and $49,400 instead of the enacted 2026 cutoffs underlying the trace. This overstated ordinary tax by about $105 while its preferential-income stacking was otherwise correct." -us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $15,000 standard deduction instead of $16,100 and then relied on projected bracket thresholds. The correct deduction leaves $93,186.13 of taxable income, and the enacted rate schedule produces $13,843.28 of ordinary tax rather than its inflated estimate." -us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly calculated $13,843.20 of ordinary tax plus about $933.90 of preferential-rate tax, or approximately $14,777.10, then discarded that result and submitted $6,710. No deduction, bracket computation, or credit in its reasoning supports the submitted value." -us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA had expired, using an $8,600 standard deduction, a restored personal exemption, and 10%/15%/25% brackets. The applicable 2026 rules instead provide a $16,100 standard deduction and 10%/12%/22% brackets over this income range." -us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model combined an incorrect $13,600 deduction-and-exemption amount with obsolete 10%/15%/25% rates. Applying the $16,100 standard deduction and enacted 10%/12%/22% schedule reduces ordinary tax to $13,843.28 before adding $933.86 of preferential-rate tax." -us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $14,930.50 exceeds the traced total even though the model identified the relevant income and standard-deduction framework. The number reflects use of incorrect 2026 deduction or bracket parameters; the enacted schedule yields $13,843.28 of ordinary tax plus $933.86 on preferential income." -us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model explicitly assumed expiration of the TCJA, reinstatement of personal exemptions, and a reduced standard deduction. Those are not the applicable 2026 parameters: the single-filer standard deduction is $16,100, and the ordinary income falls within the 10%, 12%, and 22% brackets." -us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset $8,300 standard deduction, a personal exemption, and reverted 10%/15%/25% brackets. The applicable calculation uses the $16,100 standard deduction, no personal exemption, and the enacted 10%/12%/22% schedule." -us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly reduced income using an $8,300 standard deduction plus a $5,050 personal exemption and taxed ordinary income under sunset-era parameters. The correct 2026 deduction is $16,100 with no personal exemption, producing $86,960.37 of ordinary taxable income and $13,843.28 of ordinary tax." -us,scenario_075,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $15,400 standard deduction and estimated bracket thresholds rather than the enacted $16,100 deduction and applicable schedule. That left ordinary taxable income $699.63 too high and overstated the total tax by $216.95." -us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model recognized that pension and wage income generate ordinary tax and that preferential-income tax is smaller, but its $9,468 submission omits a substantial portion of the tax produced by the progressive schedule. Taxable income is $93,186.13, with $86,960.37 taxed as ordinary income for $13,843.28 before the $933.86 preferential-rate component." -us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model improperly relied on itemized medical deductions and unspecified nonrefundable credits to reduce the liability to $1,446. The listed $600 of medical expenses is below the 7.5%-of-AGI floor, the $16,100 standard deduction applies, and no nonrefundable credit reduces the resulting tax." -us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model provided no computation supporting $13,361 and understated the traced $13,843.28 ordinary tax even before adding $933.86 of tax on net capital gain and qualified dividends. Its answer therefore omits or misapplies part of the ordinary-rate schedule or the preferential-rate component." -us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed a 2026 TCJA sunset, restored a personal exemption, reduced the standard deduction, and applied 10%/15%/25% brackets. The applicable rules instead use a $16,100 standard deduction, no personal exemption, and 10%/12%/22% ordinary rates at this income." -us,scenario_075,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an $8,522 post-sunset standard deduction and obsolete 10%/15%/25% ordinary brackets. The $16,100 deduction reduces taxable income to $93,186.13, and the enacted brackets produce $13,843.28 of ordinary tax rather than $17,916." -us,scenario_075,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so the required numeric result was missing." -us,scenario_075,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model invented approximately $18,608 of nonrefundable elderly or disability credits after first overstating taxable income and ordinary tax. At AGI of $109,286.13, no elderly-or-disabled credit offsets the liability, and no other listed fact creates a nonrefundable credit of that amount." -us,scenario_075,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a $15,750 standard deduction and 2025-style bracket thresholds of $11,925 and $48,475. The enacted 2026 deduction is $16,100 and the applicable thresholds yield $13,843.28 of ordinary tax; its preferential-rate stacking of all $6,226 at 15% was otherwise correct." +us,scenario_074,head_medicaid_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model understated Medicaid MAGI by counting only the IRA distribution plus a taxable portion of Social Security and pension income. The engine’s MAGI calculation is 2.13 times FPL, which exceeds Louisiana’s 138% FPL expansion-adult limit and leaves no qualifying Medicaid category." +us,scenario_074,head_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model incorrectly constructed MAGI as the IRA distribution, state tax refund, and exactly half of SSDI, producing $20,230. PolicyEngine calculates MAGI at 2.13 times FPL, above the expansion limit, so the adult expansion pathway does not apply." +us,scenario_074,head_medicaid_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model treated receipt of SSDI as sufficient for a disability-based Medicaid pathway and also understated MAGI by broadly excluding Social Security benefits. SSDI does not itself confer SSI-related Medicaid eligibility, the person receives no SSI, and engine MAGI of 2.13 times FPL exceeds the expansion-adult limit." +us,scenario_074,head_medicaid_eligible,gpt-5.4-mini,llm_error,health_coverage,False,"The model labeled the reported sources as producing very low countable income without applying the engine’s MAGI calculation. That calculation yields 2.13 times FPL, above Louisiana’s expansion-adult limit, and no other eligibility category applies." +us,scenario_074,head_medicaid_eligible,gpt-5.6-terra,llm_error,health_coverage,False,"The model asserted that taxable income was within the adult Medicaid limit, but the relevant engine MAGI equals 2.13 times FPL. This exceeds the 138% FPL expansion threshold, and the person qualifies through no alternative Medicaid pathway." +us,scenario_074,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted constant full-year SSDI receipt into completion of Medicare’s 24-month disability waiting period. Constant status during 2026 supplies no entitlement history before 2026, so the age-54 head remains ineligible." +us,scenario_074,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model asserted that already receiving SSDI proves the 24-month Medicare waiting period has elapsed. SSDI receipt does not establish its start date or two prior years of entitlement, and the head does not meet the age-65 pathway." +us,scenario_074,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated year-long SSDI status as satisfying the 24-month Medicare waiting period. A single full tax year is not 24 months of established entitlement, and no earlier SSDI history was provided." +us,scenario_074,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model inferred completion of the 24-month disability waiting period from SSDI income being constant during the year. The facts establish neither a pre-2026 entitlement date nor age-based eligibility, because the head is 54." +us,scenario_074,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model applied a nonexistent rule that SSDI benefits immediately confer Medicare eligibility under PolicyEngine. The under-65 disability pathway requires additional entitlement history that was not listed, while the head also fails the age-65 test." +us,scenario_074,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model explicitly assumed that SSDI was ongoing before the stated tax year and therefore that the 24-month waiting period had elapsed. The prompt forbids inferring that unlisted start date, and constant status throughout 2026 does not supply it." +us,scenario_074,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,The model treated SSDI receipt alone as sufficient for Medicare eligibility. It omitted both the age-65 requirement and the separate waiting-period condition for an under-65 disability pathway. +us,scenario_074,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model equated Social Security disability income with current Medicare eligibility. SSDI receipt does not establish completion of the disability-entitlement waiting period, and the head is only 54." +us,scenario_074,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model incorrectly made SSDI income automatically qualifying for Medicare. The facts provide no completed waiting period for disability-based eligibility, and age 54 does not satisfy age-based eligibility." +us,scenario_074,head_medicare_eligible,gemini-3.6-flash,llm_error,age_disability,False,The model treated receipt of Social Security disability income as an automatic Medicare pathway. It failed to require established completion of the disability waiting period for this 54-year-old head. +us,scenario_074,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model misstated PolicyEngine’s rule as Medicare eligibility for every SSDI recipient regardless of age. SSDI income alone does not establish the required under-65 entitlement conditions, and the head fails the age threshold." +us,scenario_074,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,The model acknowledged that age 54 does not qualify but then treated disability income as sufficient under the policy rules. It supplied no completed waiting period or other listed fact establishing the under-65 Medicare pathway. +us,scenario_074,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model assumed that receiving SSDI during the year establishes disability-based Medicare eligibility. The required entitlement history is absent, and the instruction bars inferring it from the income entry." +us,scenario_074,head_medicare_eligible,grok-4.3,llm_error,age_disability,False,The model declared that full-year SSDI receipt satisfies a 24-month waiting period. Twelve months of constant status in 2026 neither equals nor proves 24 months of prior disability entitlement. +us,scenario_074,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,The model cited the statutory disability waiting period but treated SSDI receipt as proof that it was completed. No SSDI entitlement date or elapsed 24-month period was supplied for the age-54 head. +us,scenario_074,head_medicare_eligible,grok-build-0.1,llm_error,age_disability,False,"The model expressly assumed the 24-month waiting period was met because the disability benefits were ongoing. That pre-year duration is an unlisted fact the prompt prohibits inferring, leaving the age-54 head ineligible." +us,scenario_074,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,The model treated Social Security disability income as a qualifying Medicare status by itself. It omitted the distinct entitlement-duration requirement for an under-65 beneficiary and the head does not qualify by age. +us,scenario_074,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model used constant full-year facts to infer that the disability waiting period had already been completed. Constancy within 2026 provides no pre-2026 entitlement history, so it cannot establish Medicare eligibility for a 54-year-old." +us,scenario_074,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,The model applied employee payroll-tax rates to non-wage income despite acknowledging that the household's income came from Social Security disability. It failed to set unlisted wages to zero and therefore invented a covered-earnings base that the household facts do not contain. +us,scenario_074,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required output contract." +us,scenario_075,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s written calculation reaches the correct $14,777 total and then submits $12,523 without any supporting computation. Its numeric output contradicts its own ordinary-income tax of $13,843 plus $934 of preferential-rate tax." +us,scenario_075,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"The model treated a 62-year-old as qualifying for an age-65 additional standard deduction and then invented a $5,252 disability adjustment. Disability alone does not create either deduction, so the applicable standard deduction is $16,100 and no such disability subtraction reduces the tax." +us,scenario_075,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used estimated bracket cutoffs of $12,150 and $49,400 instead of the applicable 2026 cutoffs reflected in the trace. This overstated ordinary tax as $13,948 rather than $13,843.28; the $933.86 preferential-rate component was otherwise handled correctly." +us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model initially used a $15,000 standard deduction and then substituted projected bracket thresholds, rather than applying the enacted 2026 $16,100 deduction and applicable rate schedule. Those errors raised ordinary taxable income and produced $14,153.20 of ordinary tax instead of $13,843.28." +us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model explicitly derived the correct components—$13,843.20 of ordinary tax and about $933.90 of capital-gains tax—then discarded that result and submitted $6,710 after an unexplained recalculation. Nothing in the household facts or federal calculation supports that reduction." +us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed the pre-TCJA structure returned in 2026, using an $8,600 standard deduction, a personal exemption, and 10%/15%/25% brackets. The applicable calculation instead uses a $16,100 standard deduction, no personal exemption, and the 10%/12%/22% ordinary brackets at this income." +us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model combined an understated $13,600 deduction with obsolete 10%/15%/25% brackets. Applying the $16,100 standard deduction and the applicable 10%/12%/22% schedule yields $13,843.28 of ordinary tax, not $17,315." +us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $14,930.50 exceeds the trace result after the correct $16,100 standard deduction and preferential-rate stacking. The answer implies different deduction or bracket parameters; the correct components are $13,843.28 of ordinary tax and $933.86 of capital-gains tax." +us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed expiration of the TCJA individual provisions and reinstatement of personal exemptions. The applicable 2026 computation retains the $16,100 single standard deduction, no personal exemption, and the 10%/12%/22% ordinary rate schedule at this income." +us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset $8,300 standard deduction, a $5,150 personal exemption, and reverted 10%/15%/25% brackets. The applicable rules use a $16,100 standard deduction, no personal exemption, and produce $13,843.28 of ordinary tax before adding $933.86 on preferential income." +us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used an $8,300 standard deduction, a $5,050 personal exemption, and post-sunset ordinary brackets. Those are not the applicable 2026 parameters; the correct ordinary taxable income is $86,960.37 and its tax is $13,843.28." +us,scenario_075,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,400 standard deduction instead of $16,100, overstating taxable income by $700. It also used estimated bracket thresholds, causing ordinary tax of $14,060.20 rather than $13,843.28." +us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model’s $9,468 answer omits $5,309.15 of the tax generated by the applicable ordinary brackets and preferential-rate stacking. With $86,960.37 of ordinary taxable income, ordinary tax alone is $13,843.28, before the additional $933.86 capital-gains tax." +us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated $600 of medical spending as supporting itemized deductions that drastically reduce tax. Medical expenses are deductible only above the applicable AGI floor and itemizing would not exceed the $16,100 standard deduction, so these expenses do not reduce taxable income." +us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model’s unexplained $13,361 total is below the $13,843.28 tax on ordinary income alone. Correct preferential-rate stacking then adds $933.86, producing $14,777.15 rather than the submitted amount." +us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly imposed a TCJA sunset, using an approximately $8,344 standard deduction, a personal exemption, and 10%/15%/25% brackets. The applicable rules instead use the $16,100 standard deduction, no personal exemption, and the 10%/12%/22% ordinary brackets." +us,scenario_075,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset $8,522 standard deduction and obsolete 10%/15%/25% brackets, while also omitting the personal exemption its assumed sunset regime would entail. Under the applicable rules, taxable income is $93,186.13 and ordinary tax is $13,843.28, not $17,916." +us,scenario_075,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. This is a missing-output contract failure rather than a tax computation. +us,scenario_075,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model invented roughly $18,608 of nonrefundable senior or disability credits and subtracted them from its tax. At age 62 with AGI of $109,286.13 and $90,020 of taxable pension income, no such credit offsets the liability; the trace contains no nonrefundable credit reduction." +us,scenario_075,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used a $15,750 standard deduction rather than the applicable $16,100 deduction and relied on 2025-style bracket thresholds. That overstated ordinary taxable income by $350 and produced $14,122.20 of ordinary tax instead of $13,843.28." +us,scenario_075,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model wrongly included the $2,240 tax-exempt pension in AGI, incorrectly treated the 62-year-old as over 65, and misstated the income sum as $131,866. It then contradicted its own statement that the elderly-or-disabled credit was zero by reducing a claimed $20,146 tax liability to zero." us,scenario_075,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_075,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated listed disability status as sufficient to trigger a Florida Medicaid disability pathway and did not apply the pathway-specific categorical and income tests. The reference trace shows medicaid_category = NONE, SSI received = 0, and MAGI income at 6.85 x FPL, so the disabled 62-year-old head fails all Medicaid eligibility pathways and is not eligible." -us,scenario_075,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,The model treated the generic disability indicator as equivalent to Medicare eligibility through SSDI. It failed to require SSDI entitlement and completion of the applicable Medicare waiting period for a person under 65. -us,scenario_075,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model inferred both SSDI entitlement and 24 months of entitlement from the disability indicator, despite those unlisted facts being false. It also incorrectly claimed that taking Social Security retirement benefits at age 62 produces Medicare eligibility before age 65." -us,scenario_075,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model applied a rule that all disabled individuals qualify for Medicare. Disability alone does not establish the required under-65 Medicare pathway through qualifying SSDI entitlement, ESRD, or ALS." -us,scenario_075,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model incorrectly asserted that PolicyEngine converts a disability flag directly into Medicare eligibility. At age 62, the head lacks any stated qualifying SSDI entitlement, ESRD, or ALS pathway." -us,scenario_075,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model invented an age-62 threshold for disability-based Medicare eligibility and treated proximity to age 65 as sufficient. Medicare's ordinary age threshold is 65, while under-65 eligibility requires a separately established qualifying pathway." -us,scenario_075,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,The model treated the disability indicator itself as the complete disability pathway to Medicare. It omitted the required qualifying benefit entitlement or medical-status conditions for eligibility before age 65. -us,scenario_075,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model asserted an age-62 PolicyEngine eligibility rule that does not apply. The head is below the age-65 threshold, and no qualifying under-65 Medicare status is listed." -us,scenario_075,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,The model misrepresented PolicyEngine's rule as granting Medicare eligibility to every disabled person under 65. The disability flag does not supply SSDI entitlement and its waiting period or another qualifying Medicare pathway. -us,scenario_075,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model equated being listed as disabled with satisfying Medicare's under-65 eligibility requirements. No SSDI entitlement, qualifying waiting period, ESRD, or ALS status is present." -us,scenario_075,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model reduced the rule to age 65 or disability and therefore made disability automatically sufficient. Under-65 Medicare eligibility requires a qualifying disability-related entitlement or specified medical condition, none of which is supplied by the generic disability flag." -us,scenario_075,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model identified the right payroll-tax components but rounded the final liability too aggressively. Applying 6.2% Social Security and employee Medicare to $12,656 of wages yields $968.21, not $966.00." -us,scenario_075,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model named the correct wage base and rates but made an arithmetic error. Social Security tax on $12,656 is about $784.69 and employee Medicare tax is about $183.52, totaling $968.21 rather than $1,054.00." -us,scenario_075,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable payroll_tax value or an explanation. This is a missing-output failure rather than a substantive payroll-tax computation. -us,scenario_075,payroll_tax,minimax-m3,llm_error,other,False,"The model computed the correct components, $784.67 of Social Security tax and $183.51 of Medicare tax, then replaced their $968.18 sum with an unsupported rounded value of $989. Payroll tax should retain the computed component sum, producing $968.21 after PolicyEngine's exact rounding." -us,scenario_076,child2_wic_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model invented an Idaho WIC age pathway for children through age 8; WIC child eligibility does not cover an 8-year-old child. It also made a scale error in the income test by treating $173,820 as 184% of a roughly $23,030 poverty line, when that income is many times the poverty line and exceeds the 185% WIC limit." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model's stated components produce roughly the reference result, but it abandoned that calculation and substituted an unexplained $29,226 tentative tax. The correct 2026 head-of-household bracket calculation on $139,270 is $24,211.80, and subtracting $4,600 of credits yields $19,611.80." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $9,600 of child support in gross income and deducted $6,480 of employer-sponsored premiums from wages, then used an invented $26,050 standard deduction. Child support is nontaxable, the stated wages remain $160,000, and the applicable deductions are the $24,150 standard deduction plus $800 QBI deduction." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the mortgage balance and an assumed interest rate, plus estimated state-tax itemized deductions, even though those expenses were not supplied and unlisted inputs are zero. The household therefore uses the $24,150 standard deduction, not the fabricated $37,900 itemized deduction." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model fabricated mortgage interest and state-tax deductions to claim $33,200 of itemized deductions, contrary to the instruction that unlisted numeric inputs are zero. It also discarded its own $14,306 calculation and submitted an unsupported $21,625 instead of applying the $24,150 standard deduction and $800 QBI deduction." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model treated only about $1,000 of the $4,400 CTC as reducing this output because it confused the refundable-credit limit with the nonrefundable CTC used against liability. The full $4,400 offsets the ample tentative tax here, followed by the $200 CDCC." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model applied single-filer brackets and fabricated $37,300 of itemized deductions from assumed mortgage interest and SALT. The taxpayer qualifies as head of household and uses the $24,150 standard deduction because no deductible mortgage interest or SALT amount was listed." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used single-filer brackets and the single standard deduction despite identifying two qualifying children. It also reduced listed wages by employer premiums; the correct computation uses head-of-household brackets, $160,000 of wages, and $164,220 of AGI." -us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $6,480 of employer-sponsored premiums from the explicitly listed gross wages and used a $23,200 standard deduction. The trace uses $164,220 of AGI and the 2026 head-of-household standard deduction of $24,150." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a TCJA-expiration regime with personal exemptions, a reduced standard deduction, and a $1,000-per-child CTC. For 2026 the applicable computation uses a $24,150 head-of-household standard deduction, no personal exemptions, and a $2,200 credit for each child." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies incorrect 2026 bracket or credit parameters after starting from the correct $164,220 AGI. Taxable income is $139,270 after the $24,150 standard deduction and $800 QBI deduction, producing $24,211.80 before subtracting $4,600 of nonrefundable credits." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied an obsolete post-sunset regime with personal exemptions and pre-TCJA brackets, and it omitted the child tax credit entirely. The 2026 rules instead produce $24,211.80 of tentative tax and allow $4,400 of CTC plus $200 of CDCC." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model wrongly declared the CTC fully phased out at this income and applied obsolete sunset deductions and exemptions. AGI of $164,220 is below the $200,000 head-of-household phaseout threshold, so the full $4,400 CTC reduces liability." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer's reference to income after pre-tax health-premium deductions shows that it reduced the explicitly stated $160,000 gross wages by the $6,480 employer-sponsored premium. The trace retains all $160,000 of wages, yielding $164,220 of AGI before the standard and QBI deductions." -us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced gross wages by $6,480 of employer-sponsored premiums and used a $23,100 standard deduction. The correct inputs are $164,220 of AGI, a $24,150 standard deduction, and an $800 QBI deduction." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The submitted $14,143 does not follow from the stated head-of-household computation and implies a large unreported deduction or credit. The specified derivation yields $24,211.80 before credits and $19,611.80 after the $4,400 CTC and $200 CDCC." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model explicitly treated $9,600 of child support as taxable income. Child support is excluded, so AGI is $164,220 rather than $173,820, and the resulting liability after the applicable deductions and credits is $19,611.80." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the explicitly listed gross wages by the $6,480 employee ESI premium, lowering AGI to $157,740. The correct AGI is $164,220, and the full $4,400 CTC rather than $4,000 also applies." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model used married-filing-jointly status and a $32,200 standard deduction even though no spouse is listed. The head instead files as head of household with a $24,150 standard deduction." -us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The unexplained $23,250 does not implement the specified deduction-and-credit sequence. The correct sequence produces $139,270 of taxable income, $24,211.80 of tentative tax, and $19,611.80 after $4,600 of nonrefundable credits." -us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used the wrong standard deduction and bracket thresholds, producing $24,570 of tentative tax instead of $24,211.80. It also allowed only $4,000 of CTC rather than the 2026 amount of $4,400." -us,scenario_076,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied obsolete pre-TCJA brackets, personal exemptions, and a reduced standard deduction, while omitting the CTC. The applicable 2026 rules use the $24,150 head-of-household standard deduction and provide $4,400 of CTC." -us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. It therefore failed the required structured-output contract before any substantive tax calculation could be evaluated. -us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly identified AGI, taxable income, and both credits, but computed 2026 bracket tax as $24,332.80 instead of $24,211.80. Applying the correct head-of-household bracket thresholds reduces its result by $121 to $19,611.80." -us,scenario_076,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented $1,815 of rental-property depreciation even though no depreciation expense was listed and unlisted numeric inputs are zero. Rental income remains $4,000, the QBI deduction is $800, and AGI is $164,220." -us,scenario_076,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model used married-filing-jointly brackets and a married standard deduction despite the absence of a spouse. The taxpayer files as head of household, and the model also understated the CTC at $4,000 instead of $4,400." -us,scenario_076,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated the ACTC phase-in and per-child cap as generating a refund without first determining how much CTC remained after offsetting federal income tax liability. The household's liability absorbs the available CTC nonrefundably, so refundable CTC is zero, not $2,400; it also applied the married-filing-jointly phaseout threshold to a head-of-household filer." -us,scenario_076,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model directly counted the $1,700-per-child ACTC cap as a refundable credit. That cap limits the refundable portion only after computing unused CTC, and this household's tax liability uses the CTC nonrefundably, leaving refundable CTC of zero." -us,scenario_076,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output for federal_refundable_credits, violating the required structured-output contract." -us,scenario_076,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly computed $9,920 of Social Security tax and $2,320 of Medicare tax and explicitly totaled them to $12,240, but submitted only the Social Security component as its numeric value. It failed to carry its own Medicare calculation into the final output." -us,scenario_076,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model made an arithmetic error when adding its correctly calculated components: $9,920 plus $2,320 equals $12,240, not $11,760." -us,scenario_076,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model made an arithmetic error when adding its correctly calculated components: $9,920 plus $2,320 equals $12,240, not $11,808." -us,scenario_076,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model reduced payroll-taxable wages from $160,000 to $153,520 by subtracting the $6,480 employer-sponsored insurance premium. The facts state an employer premium, not a Section 125 employee contribution, so Social Security and Medicare taxes apply to the full $160,000." -us,scenario_076,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the $6,480 employer-sponsored insurance premium as a pretax employee deduction and calculated FICA on $153,520. The premium is employer-paid under the supplied input and does not reduce the head's $160,000 payroll-tax base." -us,scenario_076,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $6,480 employer-sponsored insurance premium from W-2 wages before applying the 7.65% employee FICA rate. The supplied employer premium is not an employee pretax contribution, so the applicable base remains $160,000." -us,scenario_076,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model incorrectly classified the $6,480 employer-sponsored insurance premium as a pretax deduction from payroll-taxable wages. Social Security and Medicare taxes are calculated on the full $160,000, yielding $12,240." -us,scenario_076,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model invented a Section 125 employee salary-reduction treatment for the $6,480 employer-sponsored insurance premium. The input identifies an employer premium, so it does not reduce the $160,000 wage base for Social Security or Medicare tax." -us,scenario_076,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly calculated $9,920 of Social Security tax and $2,320 of Medicare tax but then replaced their $12,240 sum with an unsupported $10,100 estimate. No cap or combination rule reduces those stated components to $10,100." -us,scenario_076,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model incorrectly treated the $6,480 employer-sponsored insurance premium as an employee pretax deduction from FICA wages. Because the stated amount is an employer premium, employee Social Security and Medicare taxes apply to all $160,000 of wages." -us,scenario_076,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, violating the required structured-output contract." -us,scenario_076,self_employment_tax,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $4,000 of rental income as net earnings from self-employment and invented unlisted rental deductions before applying the 15.3% SECA rate. Rental income is not self-employment income on these facts, and unlisted expenses are zero under the prompt, so no self-employment tax base exists." -us,scenario_076,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable self_employment_tax output or supporting explanation. The required output was $0 because the household has no listed self-employment income, leaving no net earnings from self-employment to tax." -us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $1,000 household and dependent care expense deduction, leaving taxable income at $139,270 instead of $138,270. It also replaced Idaho's applicable 2026 rate schedule with a simple 5.3% calculation and an unsupported $410 child-credit subtraction." -us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model treated the head-of-household filer as single, included nontaxable child support in an intermediate income calculation, and invented a $9,380 Idaho adjustment. It also failed to apply the $24,150 head-of-household standard deduction, the $1,000 care-expense deduction, and the $800 QBI deduction that produce $138,270 of taxable income." -us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an obsolete 5.695% rate and an incorrect standard deduction, omitted both the $1,000 care-expense deduction and $800 QBI deduction, and inconsistently subtracted a grocery credit. Idaho's 2026 head-of-household schedule applied to $138,270 yields $6,806.79." -us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used an obsolete 5.695% rate, an incorrect $22,500 standard deduction, and a fabricated filing threshold. It also omitted the $1,000 care-expense deduction and did not consistently apply the $800 QBI deduction, so it never reached the $138,270 tax base." -us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an approximate $24,500 standard deduction, omitted the $1,000 care-expense deduction and $800 QBI deduction, and introduced an unsupported child-credit adjustment. The specified deductions produce taxable income of $138,270 before Idaho's 2026 schedule is applied." -us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model classified the filer as single and invented three Idaho personal exemptions, reducing the wrong income base with deductions that do not belong in this derivation. It also used an obsolete 5.695% rate and a $100 credit rather than applying the 2026 head-of-household schedule to $138,270." -us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used a roughly $16,000 deduction instead of the $24,150 head-of-household standard deduction and omitted the $1,000 care-expense and $800 QBI deductions. It then used an obsolete 5.695% rate and added an unexplained amount to its own computed tax." -us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model started from an unexplained $157,740 income figure rather than $164,220 and used an incorrect $23,200 standard deduction. It omitted the $1,000 care-expense deduction and $800 QBI deduction and applied an incorrect 5.8% flat rate." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model asserted taxable income of $130,867 instead of deriving $138,270 from the listed deductions. It then applied an incorrect 5.8% flat rate and subtracted an unsupported $410 child tax credit." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies a generic effective-rate estimate rather than the traced calculation. The correct computation derives $138,270 of taxable income after the $1,000 care-expense, $24,150 standard, and $800 QBI deductions and applies Idaho's 2026 head-of-household schedule to obtain $6,806.79." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an unexplained taxable-income base of $129,750 instead of $138,270 and applied the obsolete 5.695% rate as a flat multiplier. It failed to reproduce the specified deduction sequence and the applicable 2026 head-of-household schedule." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied an incorrect 5.8% flat rate and subtracted unspecified child credits. The traced result instead applies Idaho's 2026 head-of-household schedule to taxable income of $138,270 without those invented adjustments." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model supplied no computation supporting $6,985 beyond a generic reference to deductions and credits. The required derivation produces a $138,270 tax base and $6,806.79 under Idaho's 2026 head-of-household schedule." -us,scenario_076,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $22,500 standard deduction rather than $24,150 and omitted the $1,000 care-expense deduction and $800 QBI deduction, overstating taxable income as $141,720. It also used the obsolete 5.695% rate instead of Idaho's applicable 2026 schedule." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model gave no specific deduction or rate calculation supporting $6,494. The traced calculation yields taxable income of $138,270 and then $6,806.79 under Idaho's 2026 head-of-household schedule." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked unspecified itemized medical assumptions even though the calculation uses the $24,150 standard deduction. It failed to follow the actual sequence of the $1,000 care-expense deduction, standard deduction, and $800 QBI deduction to reach $138,270." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used a $23,350 standard deduction, omitted the $1,000 care-expense deduction and $800 QBI deduction, and therefore overstated taxable income as $140,870. It also subtracted two unsupported $173 child credits instead of directly applying Idaho's 2026 head-of-household schedule to $138,270." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model asserted a $132,020 tax base without deriving it from the household's $164,220 AGI and the specified deductions. It also subtracted an unsupported $440 child credit rather than applying the 2026 head-of-household schedule to $138,270." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model imposed an unsupported $4,930 zero-rate threshold and subtracted two $205 child credits. The applicable Idaho schedule operates on the correctly derived $138,270 tax base and produces $6,806.79 before refundable credits." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model omitted the $1,000 household and dependent care expense deduction, leaving taxable income at $139,270 rather than $138,270. It then treated 5.3% as a simple flat multiplier instead of applying Idaho's 2026 head-of-household schedule." -us,scenario_076,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model substituted a generic 4% effective-rate approximation for Idaho's statutory 2026 computation and did not identify the applicable deductions. The actual deduction sequence yields $138,270 of taxable income and the head-of-household schedule yields $6,806.79." -us,scenario_076,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used the obsolete 5.695% flat rate and a tax base near $141,157 rather than $138,270. It omitted the $1,000 care-expense deduction, the $800 QBI deduction, and part of the correct $24,150 standard deduction." -us,scenario_076,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model used a $12,585 standard deduction instead of the $24,150 head-of-household amount and began from an unexplained $157,740 income figure. It also omitted the $1,000 care-expense and $800 QBI deductions and used an incorrect 5.8% rate." -us,scenario_076,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. -us,scenario_076,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model omitted the $1,000 care-expense deduction and $800 QBI deduction, producing $140,070 instead of $138,270. It also invented a $2,500 threshold and subtracted two unsupported $205 child credits." -us,scenario_076,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $162,405, used an implausible $3,500 head-of-household standard deduction instead of $24,150, and omitted the $1,000 care-expense and $800 QBI deductions. It then used an obsolete 5.695% rate and invented $2,000 of child tax credits." -us,scenario_076,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model used a married-filing-jointly standard deduction even though the filer is head of household and then asserted a $96,448 tax base that does not follow from its own arithmetic. It also omitted the $1,000 care-expense and $800 QBI deductions and applied an incorrect 5.8% flat rate." -us,scenario_076,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used an outdated $120 grocery-credit amount instead of Idaho's 2026 amount of $155 per person. Three qualifying members produce $465, not $360." -us,scenario_076,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Idaho has no applicable refundable credit and omitted the refundable grocery credit. The credit provides $155 for each of the three qualifying household members regardless of this household's high income, totaling $465." -us,scenario_076,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model failed to apply the exact 2026 grocery-credit amount and substituted first $120 and then $170 per person. The governing amount is $155 for each of three qualifying members, yielding $465, with no additional refundable credit." -us,scenario_076,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model misclassified Idaho's grocery credit as nonrefundable or refundable only in limited circumstances. It is counted here as a refundable $155-per-person credit, and all three members qualify for the full year, producing $465." -us,scenario_076,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model wrongly treated the Idaho grocery credit as nonrefundable or unavailable at this income. The refundable credit applies to all three qualifying members at $155 each, totaling $465." -us,scenario_076,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly classified Idaho's grocery credit as nonrefundable. PolicyEngine counts it as a refundable credit of $155 for each of the three full-year qualifying members, totaling $465." -us,scenario_076,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model acknowledged that the grocery credit is refundable but then excluded it from state_refundable_credits and incorrectly invoked the household's income. The credit belongs in this output and equals $155 for each of three qualifying members, or $465." -us,scenario_076,state_refundable_credits,deepseek-v4-pro,llm_error,other,False,"The model substituted an Idaho child tax credit of $205 per child for the applicable refundable grocery credit. The result comes from three qualifying people—not two children—at $155 each, totaling $465." -us,scenario_076,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model identified the correct grocery-credit pathway but used $120 per person instead of the 2026 amount of $155. Three full-year qualifying members therefore receive $465, not $360." -us,scenario_076,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly treated income as disqualifying the household from all Idaho refundable credits. Idaho's refundable grocery credit applies at $155 for each of the three qualifying members, totaling $465." -us,scenario_076,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model omitted Idaho's refundable grocery credit by incorrectly concluding that no credit applies at this income level. All three household members qualify for $155 each, producing $465." -us,scenario_076,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used a $100 per-person grocery-credit amount instead of the 2026 amount of $155. Applying $155 to all three members yields $465 rather than $300. -us,scenario_076,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model incorrectly denied eligibility for Idaho's refundable grocery credit. Each of the three household members qualifies for all 12 months at an annual amount of $155, totaling $465." -us,scenario_076,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly counted three members but used an outdated $100 grocery-credit amount. Idaho's 2026 amount is $155 per qualifying member, so the total is $465." -us,scenario_076,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model omitted the Idaho refundable grocery credit despite the household facts establishing three qualifying full-year members. At $155 per member, the credit is $465." -us,scenario_076,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly applied a generic high-income cutoff to Idaho refundable credits. The grocery credit is not eliminated by this household's income and provides $155 for each of three qualifying members, totaling $465." -us,scenario_076,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model identified and counted the grocery credit correctly but used $120 per person instead of $155 for 2026. The three-person total is $465, not $360." -us,scenario_076,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable Idaho credit applies. The refundable grocery credit applies to all three household members at $155 each, producing $465." -us,scenario_076,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used an outdated $120 per-person grocery-credit amount. The 2026 amount is $155 for each of three qualifying members, yielding $465." -us,scenario_076,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model omitted Idaho's applicable refundable grocery credit. Three qualifying household members receive $155 each for a total of $465. -us,scenario_076,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model correctly counted the head and two dependents but used $100 per person instead of Idaho's 2026 amount of $155. The correct three-person total is $465. -us,scenario_076,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model invented an income phaseout that fully eliminates Idaho's grocery credit for this household. The credit remains $155 for each of the three full-year qualifying members, totaling $465." -us,scenario_076,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, so it failed the required output contract." -us,scenario_076,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model identified the grocery credit and the correct three-person count but used $120 per person. Idaho's 2026 amount is $155 per member, producing $465." -us,scenario_076,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly treated the grocery credit as dependent on a special listed trigger or low income. All three household members qualify for the refundable $155 credit for the full year, totaling $465." -us,scenario_076,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model focused on earned-income-credit eligibility and omitted Idaho's separate refundable grocery credit. The grocery credit supplies $155 for each of the three qualifying household members, totaling $465." -us,scenario_077,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's reasoning computes the required result—$6,906 of taxable income taxed at 10%, or about $691—but it submits $951 instead. Its numeric output contradicts its own completed calculation." -us,scenario_077,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracts the $8,389 employer-sponsored insurance premium from the reported wage input and uses a $14,600 standard deduction instead of $16,100. It then submits $1,402 despite its own final calculation yielding about $2, so the submitted value also contradicts its reasoning." -us,scenario_077,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derives approximately $691 from $23,006 of AGI, the $16,100 standard deduction, and the 10% rate, but submits $1,080. Its claim that inflation-adjustment rounding supports $1,080 is arithmetically incompatible with the derivation it states." -us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model uses an estimated $15,350 standard deduction rather than the 2026 single-filer amount of $16,100. That understates the deduction by $750 and overstates taxable income and tax by about $75." -us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model correctly derives roughly $691 of tax, then reduces it to zero using unspecified qualified-business-income or other adjustments that do not exist in the household facts. With no applicable nonrefundable credit or further deduction, the $690.62 liability remains." -us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applies a $15,000 standard deduction instead of the 2026 single-filer deduction of $16,100. This creates $8,006 of taxable income rather than $6,906.18." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracts the $8,389 employer-sponsored insurance premium from the reported wages and applies obsolete post-sunset concepts—a reduced standard deduction plus a personal exemption. The correct calculation uses the reported employment income, the $3,000 capital-loss deduction, and the single $16,100 standard deduction." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model uses a $15,000 standard deduction rather than $16,100 and then invents available credits that fully offset the tax. No nonrefundable credit applies, so taxable income of $6,906.18 produces $690.62 of tax." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducts the $8,389 employer-sponsored insurance premium from the wage input and assumes a TCJA sunset with a personal exemption. For 2026 the applicable standard deduction is $16,100, with no personal exemption, and AGI is $23,006.18 after the $3,000 capital-loss deduction." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model double-counts the employer-sponsored insurance premium as a deduction from reported wages and substitutes a post-TCJA-sunset standard deduction and personal exemption. The correct tax base uses $23,006.18 of AGI and the $16,100 standard deduction." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly derives $23,006 of AGI but fails to apply the full $16,100 single standard deduction. Its $965.60 answer implies only about $13,350 of deductions, rather than taxable income of $6,906.18." -us,scenario_077,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model estimates the 2026 single standard deduction at $15,450 instead of applying $16,100. The resulting $650 understatement of the deduction overstates tax by about $65." -us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly states that $26,006 of wages is below the standard deduction. After the $3,000 capital-loss deduction, $23,006.18 of AGI exceeds the $16,100 standard deduction by $6,906.18." -us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treats the wage income as fully sheltered by the standard deduction without calculating the residual taxable income. The $3,000 capital-loss deduction and $16,100 standard deduction leave $6,906.18 taxable, and no nonrefundable credit offsets its $690.62 tax." -us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model uses an $8,150 standard deduction instead of $16,100 and consequently taxes income in a second bracket. The correct taxable income is only $6,906.18, all taxed at 10%." -us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concludes that the standard deduction and capital-loss offset reduce taxable income to zero. Together the $3,000 allowable capital loss and $16,100 standard deduction leave $6,906.18 of taxable income." -us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly applies post-TCJA-sunset amounts of an $8,300 standard deduction and $5,300 personal exemption. The 2026 calculation instead uses a $16,100 standard deduction and no personal exemption, leaving $6,906.18 taxable." -us,scenario_077,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly subtracts the $8,389 employer-sponsored insurance premium from the reported wages and uses an $8,300 standard deduction. The correct AGI is $23,006.18 after only the $3,000 capital-loss deduction, followed by the $16,100 standard deduction." -us,scenario_077,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so it failed the required output contract." -us,scenario_077,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model offsets wages by the full $7,968 capital loss instead of enforcing the $3,000 annual deduction limit, uses the wrong standard deduction, and invokes nonexistent personal-exemption credits. The allowed loss produces $23,006.18 of AGI, and the $16,100 standard deduction leaves positive taxable income." -us,scenario_077,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly limits the capital-loss deduction to $3,000 but uses a $15,750 standard deduction instead of $16,100. This overstates taxable income by $350 and tax by about $35." -us,scenario_077,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly reasoned that the childless EITC phases out to zero when applying the greater of earned income or AGI, then submitted 56 anyway. Its numeric answer contradicts its own final EITC computation and leaves a residual refundable credit after the completed childless EITC phaseout." -us,scenario_077,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model treated the childless EITC as leaving a tiny residual credit at roughly $23,006 of AGI. For a single filer with no qualifying children, the 2026 childless EITC is fully phased out at this income level, so the correct refundable credit total is zero." -us,scenario_077,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model reduced earned income from $26,006 to $17,617 by subtracting employer-sponsored insurance premiums, then used that lower amount for the childless EITC phaseout. EITC earned income is wages for this household, and the greater-of-earned-income-or-AGI phaseout does not allow those premiums to preserve a residual childless EITC." -us,scenario_077,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model assigned a positive no-child EITC without applying the single-filer childless EITC completed phaseout. At this household’s wages and AGI, the no-child EITC is reduced to zero, and no other refundable federal credit pathway is available." -us,scenario_077,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model reduced earned income to $17,617 by subtracting the employer-sponsored insurance premium and then computed a residual childless EITC. The EITC phaseout uses earned income from wages and AGI under the statutory test, and this filer’s income fully eliminates the no-child EITC." -us,scenario_077,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model used $17,617 instead of the $26,006 wage amount in the childless EITC phaseout and selected an incorrect phaseout start. Using the household’s actual earned income and AGI, the no-child EITC is fully phased out, leaving no refundable federal credits." -us,scenario_077,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented a qualifying child from the school-meal and benefit outputs even though the facts list only one 37-year-old adult. Without a qualifying child there is no Child Tax Credit or refundable additional CTC, and the remaining childless EITC is fully phased out." -us,scenario_077,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model’s explanation states the correct rule: no qualifying children, income above the childless EITC phaseout range, and no other refundable credits. Its submitted value of 111 is an internal consistency error that adds a residual refundable credit after correctly eliminating every refundable-credit component." -us,scenario_077,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model reduced earned income to $17,617 and AGI to $14,617, then applied the childless EITC phaseout to those understated figures. The household’s wages are $26,006 and the capital loss deduction does not lower the EITC phaseout base enough to avoid the completed no-child EITC phaseout." -us,scenario_077,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable value or explanation for federal_refundable_credits. This is a missing-output failure rather than a substantive tax calculation. -us,scenario_077,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated the childless EITC as a phased-in positive benefit at $26,006 of earnings. For a single filer with no qualifying children, that income is beyond the completed EITC phaseout range, so the childless EITC and total federal refundable credits are zero." -us,scenario_077,head_medicaid_eligible,claude-opus-4.8,llm_error,health_coverage,False,The model miscomputed the Medicaid expansion income test by calling income at or below the 138% FPL limit when PolicyEngine's MAGI income level is 1.44 times FPL. It also relied on a rough single-person threshold estimate and therefore incorrectly assigned the Louisiana adult MAGI category despite the engine placing the person in medicaid_category NONE. -us,scenario_077,head_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model incorrectly subtracted employer-sponsored insurance premiums from Medicaid MAGI and reduced income to about $17,617. PolicyEngine's Medicaid MAGI calculation leaves the head at 1.44 times FPL, above the Louisiana adult expansion limit, and no non-MAGI eligibility category applies." -us,scenario_077,head_medicaid_eligible,gemini-3-flash-preview,llm_error,health_coverage,False,"The model incorrectly reduced Medicaid MAGI by both employer-sponsored insurance premiums and a $3,000 capital loss deduction, producing $14,617 and placing the head below 138% FPL. PolicyEngine's computed Medicaid MAGI level is 1.44 times FPL, so the Louisiana adult expansion pathway is not available and medicaid_category remains NONE." -us,scenario_077,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,health_coverage,False,"The model used an AGI shortcut of $14,617 as if it were the Medicaid MAGI amount for Louisiana expansion eligibility. PolicyEngine's Medicaid MAGI income level is 1.44 times FPL, above the applicable MAGI limit, so the head fails the adult expansion category and has no alternative Medicaid category." -us,scenario_077,head_medicaid_eligible,gemini-3.5-flash,llm_error,health_coverage,False,"The model treated $14,617 as the head's Medicaid MAGI and compared that to the 138% FPL expansion threshold. PolicyEngine computes the relevant MAGI income level as 1.44 times FPL, which fails the MAGI expansion test, and the head qualifies through none of the other Louisiana Medicaid pathways." -us,scenario_077,head_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"The model asserted that income is below Louisiana's adult Medicaid threshold without performing the PolicyEngine MAGI category calculation. The engine's relevant MAGI income level is 1.44 times FPL, above the expansion adult limit, and the head has no SSI, dependent, aged, disabled, child, pregnancy, or other category facts that create another Medicaid pathway." -us,scenario_077,head_medicaid_eligible,grok-build-0.1,llm_error,health_coverage,False,"The model used a $14,617 MAGI figure and compared it to 138% FPL for Louisiana expansion eligibility. PolicyEngine's Medicaid MAGI income level is 1.44 times FPL, so the head is over the MAGI limit and does not enter any other Medicaid eligibility category." -us,scenario_077,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable Medicaid eligibility output or supporting explanation. The required result is not eligible because PolicyEngine computes medicaid_category NONE: MAGI income is 1.44 times FPL and no alternative Louisiana Medicaid pathway applies. +us,scenario_075,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the listed disability as sufficient to activate a Florida disabled-adult Medicaid pathway without applying that pathway's complete eligibility requirements. The head receives no SSI and the engine assigns medicaid_category NONE, so no categorical pathway exists and Medicaid eligibility is 0." +us,scenario_075,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model equated the generic disability flag with disability-based Medicare entitlement. It skipped the requirement for qualifying SSDI-based entitlement and its waiting period, while the head is only 62." +us,scenario_075,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model invented satisfaction of the 24-month SSDI-entitlement requirement from the generic disability flag, even though SSDI entitlement is unlisted and therefore false. It also incorrectly treated eligibility for Social Security retirement benefits at age 62 as conferring Medicare eligibility before age 65." +us,scenario_075,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated disability by itself as sufficient for Medicare. An under-65 person needs a qualifying Medicare entitlement pathway, and none is supplied for this 62-year-old head." +us,scenario_075,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,The model incorrectly asserted that PolicyEngine converts the disability indicator directly into Medicare eligibility. The head remains below age 65 and has no listed SSDI-based Medicare entitlement or other qualifying condition. +us,scenario_075,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model invented an age-62 threshold for disability-based Medicare eligibility and treated approaching age eligibility as sufficient. The ordinary age threshold is 65, and disability alone does not establish the separate under-65 entitlement pathway." +us,scenario_075,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model treated the disability flag as completing the under-65 Medicare pathway. It omitted the required qualifying entitlement facts, leaving the 62-year-old head below the applicable age threshold." +us,scenario_075,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model asserted without a valid rule that age 62 satisfies PolicyEngine's Medicare eligibility indicator. The head is below the age-65 threshold, and no qualifying under-65 Medicare entitlement is listed." +us,scenario_075,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,The model incorrectly reduced PolicyEngine's rule to disabled-or-age-65. A generic disability status does not itself establish Medicare entitlement for this 62-year-old head. +us,scenario_075,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model treated disability as an automatic substitute for the age requirement. It failed to require a qualifying under-65 Medicare entitlement pathway, which the household facts do not provide." +us,scenario_075,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model applied an incorrect age-65-or-disabled Boolean rule. Disability alone is not Medicare entitlement, and the 62-year-old head has no listed fact establishing the under-65 pathway." +us,scenario_075,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model inferred Medicare eligibility directly from age 62 plus disability. Age 62 is below the Medicare age threshold, and the disability flag does not establish qualifying SSDI-based or other under-65 Medicare entitlement." +us,scenario_075,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model named the correct 6.2% Social Security and 1.45% Medicare rates but miscalculated their combined application to wages. The PolicyEngine component amounts are $784.69 and $183.52, which sum to $968.21 rather than $966." +us,scenario_075,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model applied the stated FICA rates incorrectly: 6.2% plus 1.45% on $12,656 does not produce $1,054. The component calculation yields $784.69 of Social Security tax and $183.52 of Medicare tax, totaling $968.21." +us,scenario_075,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." +us,scenario_075,payroll_tax,minimax-m3,llm_error,other,False,"The model's own component calculation produced $968.18 before engine-level precision, but it then changed that result to $989 under the label of rounding. Rounding cannot turn approximately $968.2 into $989; the PolicyEngine components total $968.21." +us,scenario_076,child2_wic_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model falsely extended Idaho WIC eligibility through age 8 even though WIC child eligibility ends at the fifth birthday. It also divided or expressed the income ratio incorrectly: using its own $23,030 poverty guideline, $173,820 is about 755% of poverty and exceeds the 185% limit by a wide margin; medical expenses do not establish WIC eligibility or cure either failure." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model's stated bracket calculation gives roughly $24,321 before credits, but it then replaced that result with an unexplained $29,226 tentative tax. Applying the 2026 brackets to $139,270 yields $24,211.80, from which $4,600 of nonrefundable credits must be subtracted." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $9,600 of child support in gross income, deducted the $6,480 employer-sponsored insurance premium from wages, omitted the $800 QBI deduction, and used an invented $26,050 standard deduction. It also allowed only $4,000 of CTC and omitted the $200 CDCC from its final answer instead of subtracting $4,600 in total credits." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model fabricated mortgage interest from the reported mortgage balance and an assumed interest rate, then fabricated a SALT deduction from an estimated Idaho tax liability. No mortgage-interest payment or deductible SALT input was listed, so the computation uses the $24,150 standard deduction and $800 QBI deduction." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest by multiplying the balance by an assumed rate and added estimated SALT, even though neither deductible payment was provided. It then submitted $21,625 despite its own itemized-deduction calculation producing $14,306, abandoning both its derivation and the required standard-deduction computation." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly treated only about $1,000 of the $4,400 CTC as nonrefundable credit usable against current tax. Because tentative tax exceeds the entire credit, all $4,400 reduces tax before refundable credits, followed by the $200 CDCC." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used single filing status and fabricated $27,300 of mortgage interest plus $10,000 of SALT from facts that supplied no deductible payments. The household qualifies as head of household and uses the $24,150 standard deduction, not those invented itemized deductions." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model applied the single standard deduction and single tax brackets even though the head supports two qualifying children and files as head of household. It also deducted the listed ESI premium from wages without an input establishing pretax exclusion and used only $4,000 rather than $4,400 of CTC." +us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced wages by the $6,480 ESI premium even though the gross-wage input remains $160,000 for this calculation, and it used the wrong standard deduction. It also used $4,000 rather than the 2026 $4,400 CTC and omitted the $800 QBI deduction from its stated taxable-income calculation." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied an assumed TCJA sunset regime with personal exemptions, a reduced standard deduction, pre-2018-style parameters, and a $1,000-per-child CTC. The applicable 2026 rules instead provide a $24,150 head-of-household standard deduction, no personal exemptions in this derivation, and a $2,200 credit for each child." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies an incorrect deduction, bracket, or credit computation despite starting from the correct $164,220 AGI. The required steps produce $139,270 of taxable income, $24,211.80 of tentative tax, and $19,611.80 after $4,600 of nonrefundable credits." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model imposed an assumed TCJA expiration, deducted the ESI premium, and used personal exemptions and pre-TCJA tax parameters. It also omitted the $4,400 CTC entirely, subtracting only the $200 CDCC." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model incorrectly declared the CTC fully phased out at $157,740 of its own AGI calculation. The head-of-household phaseout threshold is $200,000 here, so both children generate the full $4,400 nonrefundable CTC." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer's reference to income after pretax health-premium deductions shows that it reduced the $160,000 gross-wage input by the listed ESI premium. The calculation instead uses $164,220 of AGI and then subtracts the $24,150 standard deduction, $800 QBI deduction, and $4,600 of credits." +us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $6,480 ESI premium from gross wages and used an estimated $23,100 standard deduction. It also allowed only $4,000 of CTC rather than the applicable $4,400." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The submitted $14,143 does not follow from the stated head-of-household approach and implies excessive deductions or credits. The specified inputs yield $139,270 of taxable income and $24,211.80 before subtracting exactly $4,600 of nonrefundable credits." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model expressly treated $9,600 of child support as taxable income, although child support received is excluded from federal gross income. It also failed to apply the exact $24,150 standard deduction, $800 QBI deduction, and $4,600 of nonrefundable credits." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly reduced gross wages by the $6,480 ESI premium and relied on an estimated standard deduction. It also used $4,000 of CTC instead of the applicable $4,400, understating both AGI and the credit." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model used married-filing-jointly status and a $32,200 joint standard deduction even though no spouse is listed. The head qualifies for head-of-household status and receives the $24,150 head-of-household standard deduction." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The unexplained $23,250 does not reflect the required traced sequence. The correct sequence is $164,220 AGI, $139,270 taxable income, $24,211.80 tentative tax, and a $4,600 reduction for the CTC and CDCC." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an estimated $23,063 standard deduction and calculated about $24,570 of tax from resulting taxable income of $140,357. The applicable standard deduction is $24,150, taxable income is $139,270, and the 2026 brackets yield $24,211.80 before credits." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied a TCJA-expiration regime with personal exemptions and older bracket structure, while also deducting the ESI premium from wages. It then omitted the entire $4,400 CTC and subtracted only the $200 CDCC." +us,scenario_076,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model subtracted only about $1,000 of CTC despite tentative tax being sufficient to absorb the full $4,400 credit. The entire $4,400 CTC is nonrefundable credit used here, along with the $200 CDCC." +us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly derived AGI, both deductions, and $4,600 of credits, but calculated tentative tax as $24,332.80. Applying the 2026 head-of-household brackets to $139,270 yields $24,211.80, exactly $121 less." +us,scenario_076,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented $1,815 of rental depreciation even though unlisted expenses are zero, thereby reducing rental income and the associated QBI deduction. It also used an estimated $23,200 standard deduction and only $4,000 rather than $4,400 of CTC." +us,scenario_076,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model used married-filing-jointly brackets and a purported MFJ standard deduction despite the absence of a spouse. It also omitted the $800 QBI deduction and used $4,000 rather than $4,400 of CTC." +us,scenario_076,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model included nontaxable child support, inconsistently handled the ESI premium, invented dependent deductions, and asserted a CTC phaseout threshold above $400,000 while claiming income near $170,000 exceeded it. Its submitted $18,779.30 also contradicts its own stated $14,152.40 tax calculation." +us,scenario_076,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model treated the ACTC income phase-in and per-child cap as creating a refundable payment without first determining whether any CTC remained after offsetting federal income tax liability. The household’s liability absorbs the entire CTC nonrefundably, so neither its unsupported $2,400 adjustment nor any other refundable CTC remains." +us,scenario_076,federal_refundable_credits,claude-opus-5,llm_error,other,False,"The model automatically assigned the $1,700-per-child refundable cap to both children. That cap only limits an otherwise available refundable CTC; because the full CTC is used against this household’s substantial federal income tax liability, the refundable remainder is zero." +us,scenario_076,federal_refundable_credits,inkling,llm_error,other,False,"The model equated satisfying the ACTC earned-income phase-in with receiving the maximum refundable amount for each child. It omitted the prior step allocating the CTC against federal income tax liability, which uses the full credit nonrefundably and leaves zero refundable CTC." +us,scenario_076,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for federal_refundable_credits, violating the required structured-output contract." +us,scenario_076,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly calculated the $12,240 total but submitted $9,920, the Social Security component alone. Its numeric output omitted the $2,320 employee Medicare tax that its own explanation included." +us,scenario_076,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model stated the correct components, $9,920 of Social Security tax and $2,320 of Medicare tax, but added them incorrectly. Those components sum to $12,240, not $11,760." +us,scenario_076,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model stated the correct components, $9,920 of Social Security tax and $2,320 of Medicare tax, but added them incorrectly. Their sum is $12,240, not $11,808." +us,scenario_076,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages from $160,000 to $153,520 by subtracting the $6,480 employer-sponsored insurance premium. PolicyEngine applies both the 6.2% Social Security rate and 1.45% Medicare rate to the full $160,000, yielding $12,240." +us,scenario_076,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed $6,480 employer-sponsored insurance premium as a pre-tax Section 125 deduction and reduced FICA wages to $153,520. The benchmark provides gross wages of $160,000 as the payroll-tax base, producing $9,920 of Social Security tax plus $2,320 of Medicare tax." +us,scenario_076,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted the $6,480 insurance premium from wages even though the household facts do not establish that it reduces FICA wages. Applying 7.65% to the full $160,000 produces $12,240." +us,scenario_076,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model incorrectly classified the $6,480 ESI premium as a pre-tax payroll deduction and calculated FICA on $153,520. Social Security and Medicare taxes are calculated here on the full $160,000 wage input, totaling $12,240." +us,scenario_076,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model invented Section 125 treatment for the listed ESI premium and reduced payroll-tax wages by $6,480. The facts do not specify that deduction from FICA wages, so the applicable base is $160,000 and employee FICA totals $12,240." +us,scenario_076,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly computed $9,920 of Social Security tax and $2,320 of Medicare tax but then replaced their $12,240 sum with an unsupported estimate of $10,100. No cap or combination rule reduces these stated components to its submitted amount." +us,scenario_076,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model improperly assumed the $6,480 ESI premium was excluded from FICA wages and applied 7.65% to $153,520. The payroll-tax calculation uses the full $160,000 wage amount, yielding $12,240." +us,scenario_076,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll-tax output or explanation. It therefore failed the required structured-output contract rather than performing the Social Security-plus-Medicare calculation. +us,scenario_076,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly treated the $4,000 of rental income as earnings from self-employment and invented a 30% expense deduction that the prompt did not provide. Rental income is not subject to self-employment tax here, leaving a zero tax base and $0 liability." +us,scenario_076,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no self_employment_tax output or explanation, violating the required structured-output contract." +us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It omitted the $1,000 household and dependent care expense deduction, leaving taxable income at $139,270 instead of $138,270. It also improvised child-credit reductions rather than applying the 2026 Idaho head-of-household schedule to the traced taxable-income base." +us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It treated the filer as single, used a $15,000 deduction, and invented a $9,380 state adjustment. The filer is head of household, and the traced deductions produce $138,270 of taxable income before Idaho’s 2026 schedule is applied." +us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the obsolete 5.695% rate and an incorrect $23,625 standard deduction, then inconsistently subtracted a grocery credit while still reporting a higher result. The correct computation applies the 2026 head-of-household schedule to $138,270." +us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used an obsolete 5.695% rate, an estimated $22,500 standard deduction, and a fabricated exemption threshold. It also failed to reconcile its stated $7,876 calculation with its submitted $9,112 value." +us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It estimated the standard deduction, omitted the separate $1,000 care-expense and $800 QBI deductions from a coherent taxable-income derivation, and then added unspecified adjustments. The traced taxable income is $138,270, to which the 2026 Idaho schedule applies." +us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It classified the head-of-household filer as single and invented three Idaho personal exemptions, while using an obsolete 5.695% rate. Idaho taxable income instead follows from the $24,150 head-of-household standard deduction plus the $1,000 care and $800 QBI deductions." +us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It used an unsupported $16,000 deduction and then added nearly $1,000 of unexplained 'conformity nuances' that do not follow from its arithmetic. The specified deductions reduce AGI to $138,270, and the 2026 schedule yields $6,806.79." +us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $157,740 without a valid adjustment, used a $23,200 standard deduction, and omitted the traced care-expense and QBI deductions. It then applied an incorrect 5.8% flat-rate shortcut rather than the 2026 schedule." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It asserted taxable income of $130,867 instead of deriving $138,270 from the listed deductions. It also applied a 5.8% flat rate and an unsupported $410 child-credit subtraction." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $8,100 estimate does not incorporate the traced $24,150 standard deduction, $1,000 care-expense deduction, and $800 QBI deduction into the $138,270 taxable-income base. It consequently failed to apply the correct 2026 Idaho schedule to that base." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used an unexplained $129,750 taxable-income estimate rather than the traced $138,270. It also applied the obsolete 5.695% flat rate instead of Idaho’s applicable 2026 head-of-household schedule." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used an approximate 5.8% flat rate and unspecified child credits instead of the applicable 2026 Idaho head-of-household schedule. The correct schedule is applied to $138,270 after all three traced deductions." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its unexplained $6,985 estimate does not reproduce the traced taxable-income calculation or the applicable schedule. AGI of $164,220 falls to $138,270 after the $1,000, $24,150, and $800 deductions, producing $6,806.79." +us,scenario_076,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used a $22,500 standard deduction, omitted both the $1,000 care-expense deduction and $800 QBI deduction, and therefore overstated taxable income at $141,720. It compounded that error by applying the obsolete 5.695% rate." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its unsupported $6,494 answer does not follow the traced deductions and schedule. The correct derivation yields $138,270 of taxable income and $6,806.79 of tax before refundable credits." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It invoked unspecified itemized medical assumptions even though the trace uses the $24,150 standard deduction, plus the $1,000 care-expense and $800 QBI deductions. Those deductions establish a $138,270 tax base, not the base implicit in $7,450." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used a $23,350 standard deduction, omitted the $1,000 care-expense and $800 QBI deductions, and subtracted invented $173-per-child credits. The correct taxable income is $138,270 and must be processed through the 2026 Idaho schedule." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It asserted $132,020 of taxable income and a $440 child credit without deriving either from the household facts. The traced deductions yield $138,270, and the applicable schedule produces $6,806.79 without that invented shortcut." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It imposed an approximate $4,930 zero-rate threshold and subtracted two $205 child credits rather than applying the traced 2026 Idaho schedule. Those unsupported adjustments drove its result below the $6,806.79 liability on $138,270." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It stopped at $139,270 because it subtracted the standard deduction and QBI deduction but omitted the $1,000 household and dependent care expense deduction. It then treated 5.3% as a simple flat multiplication instead of applying the 2026 head-of-household schedule." +us,scenario_076,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It substituted an unexplained 4% effective-rate estimate for Idaho’s 2026 head-of-household tax schedule. Applying that schedule to the traced $138,270 taxable-income base yields $6,806.79." +us,scenario_076,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used the obsolete 5.695% rate and denied the QBI deduction, producing an overstated taxable-income base near $141,157. The trace includes the $800 QBI deduction and $1,000 care-expense deduction, leaving $138,270 for the 2026 schedule." +us,scenario_076,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used an unexplained $157,740 AGI, a $12,585 standard deduction, and a 5.8% flat rate. The actual AGI is $164,220 and the $24,150 standard deduction plus $1,000 care and $800 QBI deductions produce $138,270." +us,scenario_076,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It replaced the exact Idaho schedule with a rough 5.3% estimate over a broad $135,000–$140,000 range. The traced base is precisely $138,270, and applying the 2026 head-of-household schedule yields $6,806.79." +us,scenario_076,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It supplied no numeric output or explanation for the requested variable. +us,scenario_076,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It omitted the $1,000 care-expense deduction and $800 QBI deduction, producing $140,070 rather than $138,270 of taxable income. It also imposed a $2,500 threshold and two $205 credits that are not part of the traced computation." +us,scenario_076,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It understated AGI at $162,405, used a fictitious $3,500 head-of-household standard deduction, and subtracted invented $1,000-per-child credits. The correct deductions from $164,220 produce $138,270, followed by the 2026 Idaho schedule." +us,scenario_076,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It treated the filer as married filing jointly and used an approximate $30,000 deduction even though the household supports head-of-household status. It then asserted $96,448 of taxable income without arithmetic support and applied an incorrect 5.8% flat rate." +us,scenario_076,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It used an unexplained $121,620 federal taxable-income starting point, subtracted a fictitious $2,400 Idaho standard deduction, and applied obsolete graduated brackets. The 2026 computation instead reaches $138,270 after the traced deductions and applies the current head-of-household schedule." +us,scenario_076,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,The model used an outdated $120 grocery-credit amount instead of Idaho's 2026 amount of $155 per qualifying person. Applying $155 to the head and two children yields $465. +us,scenario_076,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Idaho has no applicable refundable credit and omitted the refundable grocery credit. The credit applies to all three household members regardless of this household's high income, producing $465." +us,scenario_076,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model vacillated between $120 and $170 per person and ultimately used the incorrect $170 amount. Idaho's 2026 grocery credit is $155 for each of the three qualifying members, totaling $465, with no additional refundable credit." +us,scenario_076,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model misclassified Idaho's grocery credit as nonrefundable or refundable only in limited circumstances. It is included as a refundable credit here at $155 for each of three qualifying household members, totaling $465." +us,scenario_076,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly treated the Idaho grocery credit as nonrefundable or unavailable at this income. The refundable credit supplies $155 for each of the three members, totaling $465." +us,scenario_076,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly classified Idaho's grocery credit as nonrefundable and therefore omitted it. All three household members receive the refundable $155 credit, producing $465." +us,scenario_076,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model acknowledged that the grocery credit is refundable but then excluded it as though income or placement elsewhere removed it. The credit remains in state_refundable_credits and equals $155 for each of three members, or $465." +us,scenario_076,state_refundable_credits,deepseek-v4-pro,llm_error,other,False,"The model substituted an inapplicable $205-per-child Idaho child tax credit for the applicable refundable grocery credit. The result comes from three qualifying people at $155 each, not two children at $205 each, so it is $465." +us,scenario_076,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly identified the refundable grocery credit but used $120 per person instead of the 2026 amount of $155. Three qualifying members therefore receive $465, not $360." +us,scenario_076,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model incorrectly treated household income as disqualifying for Idaho's refundable grocery credit. No such income exclusion removes the credit here, and three qualifying members at $155 each yield $465." +us,scenario_076,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model incorrectly concluded that this income level prevents any applicable refundable Idaho credit. The grocery credit still applies to all three members at $155 each, totaling $465." +us,scenario_076,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an outdated $100 grocery-credit amount instead of the 2026 amount of $155 per qualifying person. For three household members, the correct computation is 3 × $155 = $465." +us,scenario_076,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model omitted Idaho's refundable grocery credit by declaring the household ineligible. Each of the three members qualifies for the full year, generating $465." +us,scenario_076,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,The model correctly counted three recipients but used $100 per person instead of Idaho's 2026 $155 amount. The grocery credit is therefore $465. +us,scenario_076,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model overlooked the Idaho refundable grocery credit because it searched only for special facts indicating eligibility. Ordinary full-year residency of the head and two dependents supplies three qualifying members at $155 each, totaling $465." +us,scenario_076,state_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model applied a generic high-income cutoff to Idaho's grocery credit even though this credit is not eliminated at the household's income. All three members qualify for $155 each, producing $465." +us,scenario_076,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly identified and counted the grocery-credit recipients but used the outdated $120 amount. Idaho's 2026 amount is $155 per person, so three recipients yield $465." +us,scenario_076,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable Idaho credit applies and omitted the grocery credit. The head and both children each qualify for $155, totaling $465." +us,scenario_076,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,The model used $120 per person rather than Idaho's 2026 grocery-credit amount of $155. Multiplying the correct amount by three household members gives $465. +us,scenario_076,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model omitted Idaho's applicable refundable grocery credit. Three full-year qualifying household members receive $155 each, totaling $465." +us,scenario_076,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly counted the head and two dependents but used an outdated $100 grocery-credit amount. The 2026 amount is $155 per person, producing $465." +us,scenario_076,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model incorrectly applied a complete income phaseout to Idaho's grocery credit. The credit is not phased out for this household, and three qualifying members at $155 each receive $465." +us,scenario_076,state_refundable_credits,inkling,llm_error,thresholds_rates,False,The model used $100 per household member instead of the 2026 grocery-credit amount of $155. The three-member credit is $465. +us,scenario_076,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable state_refundable_credits value. The required output was $465, derived from three full-year qualifying members receiving $155 each." +us,scenario_076,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model correctly identified all three grocery-credit recipients but used $120 rather than the 2026 $155 amount. The correct total is 3 × $155 = $465. +us,scenario_076,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated the household's income as preventing the grocery credit from being triggered. Idaho's refundable grocery credit remains available, with $155 for each of three members totaling $465." +us,scenario_076,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model focused on earned-income-credit eligibility and failed to apply Idaho's separate refundable grocery credit. The grocery credit covers all three household members at $155 each, producing $465." +us,scenario_076,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model omitted Idaho's refundable grocery credit by declaring that no credit applied. All three members qualify for the full-year $155 amount, so refundable state credits total $465." +us,scenario_077,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own derivation correctly produces approximately $691, but it submitted $951 instead. The submitted value does not follow from its stated taxable income of $6,906 taxed at 10%." +us,scenario_077,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the listed $8,389 employer-sponsored insurance premium from the separately specified gross wages and used a $14,600 standard deduction instead of $16,100. Gross wages remain $26,006.18; after the $3,000 capital-loss deduction and $16,100 standard deduction, taxable income is $6,906.18, not the contradictory amounts in its reasoning or its unsupported $1,402 submission." +us,scenario_077,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived taxable income of about $6,906 and tax of about $691, then submitted $1,080 without any computation supporting that value. The correct 10% tax on $6,906.18 is $690.62." +us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an estimated $15,350 standard deduction instead of the 2026 single-filer amount of $16,100. That understated the deduction by $750 and raised taxable income and tax by $75." +us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly derived roughly $691 of regular tax, then erased it using unspecified qualified-business-income or other adjustments despite no qualifying facts. No nonrefundable credit or additional deduction offsets the $690.62 liability." +us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction instead of the 2026 single-filer amount of $16,100. The additional $1,100 deduction reduces taxable income from its $8,006 figure to $6,906.18 and tax from $800.60 to $690.62." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $8,389 employer-sponsored insurance premium from the stated gross wages and applied a post-sunset standard-deduction-plus-personal-exemption structure. The listed gross wages enter income as given, and the applicable 2026 deduction is the $16,100 standard deduction with no separate personal exemption." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model calculated positive taxable income and $800.60 of tentative tax under its own deduction assumption, then claimed unspecified credits fully offset it. No nonrefundable credit applies, so the tax remains $690.62 after using the correct $16,100 standard deduction." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced gross wages by the $8,389 employer-sponsored insurance premium and assumed a TCJA-sunset deduction and personal exemption. The calculation instead starts from the stated $26,006.18 of wages and applies the $3,000 capital-loss deduction and $16,100 standard deduction." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the $8,389 employer-sponsored insurance premium from gross wages and substituted a post-TCJA-sunset standard deduction plus personal exemption. The applicable computation uses the stated wages without that premium subtraction and a single $16,100 standard deduction." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly reached AGI of about $23,006 but implicitly used only about $13,350 of deductions to produce $965.60 of tax. The 2026 single standard deduction is $16,100, leaving $6,906.18 taxable and $690.62 of tax." +us,scenario_077,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $15,450 standard deduction instead of $16,100. The correct deduction leaves taxable income of $6,906.18 rather than $7,556 and produces $690.62 at the 10% rate." +us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly stated that $26,006 of wages are below the standard deduction. After the $3,000 capital-loss deduction, AGI of $23,006.18 exceeds the $16,100 standard deduction by $6,906.18, creating $690.62 of tax." +us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the wage income as fully sheltered by the standard deduction without computing AGI and taxable income. The $3,000 capital-loss deduction and $16,100 standard deduction leave $6,906.18 taxable, and the absence of nonrefundable credits leaves $690.62 due." +us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an $8,150 standard deduction and a 15% second bracket under an obsolete post-sunset tax structure. The applicable 2026 single standard deduction is $16,100, and all $6,906.18 of taxable income falls in the 10% bracket." +us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that taxable income is zero or negative. AGI is $23,006.18 after the limited $3,000 capital-loss deduction, so subtracting the $16,100 standard deduction leaves $6,906.18 taxable." +us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset $8,300 standard deduction and $5,300 personal exemption instead of the applicable $16,100 standard deduction. This overstated taxable income by $2,500 and tax by about $250." +us,scenario_077,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted the $8,389 employer-sponsored insurance premium from the stated gross wages and then used an $8,300 post-sunset standard deduction. Gross wages remain $26,006.18 for this calculation, and the applicable standard deduction is $16,100." +us,scenario_077,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so it failed the required output contract." +us,scenario_077,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model offset wages by the full $7,968 capital loss instead of enforcing the $3,000 annual deduction limit, then invoked nonexistent personal-exemption credits to reduce tax to zero. Only $3,000 offsets wages, and no nonrefundable credit applies to the resulting $690.62 liability." +us,scenario_077,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model correctly limited the capital-loss deduction to $3,000 but used a $15,750 standard deduction instead of $16,100. The extra $350 deduction lowers taxable income and tax by $35, producing $690.62." +us,scenario_077,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the standard deduction and $3,000 capital-loss deduction eliminate taxable income, then invoked unspecified nonrefundable credits. Those deductions leave $6,906.18 taxable, and no nonrefundable credit reduces the resulting $690.62 tax." +us,scenario_077,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly concluded that using $26,006 of earned income fully phases out the childless EITC, then submitted $56 despite deriving no refundable credit. It failed to carry its own zero-EITC result into the output." +us,scenario_077,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model treated the capital-loss-reduced AGI of about $23,006 as leaving a residual childless EITC. EITC phaseout uses the greater of earned income and AGI, and the $26,006 of wages fully phases out the credit." +us,scenario_077,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model replaced the stated $26,006 of gross wages with $17,617 by subtracting the $8,389 employer-sponsored insurance premium when computing EITC earned income. Using the stated wages in the childless EITC phaseout reduces the credit to zero." +us,scenario_077,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model awarded $1,184 of childless EITC even though the taxpayer’s $26,006 of wages is above the 2026 single-filer phaseout ceiling. It applied neither the childless-credit maximum nor the full phaseout at this income." +us,scenario_077,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model incorrectly used $17,617 as earned income, exactly subtracting the $8,389 insurance premium from the stated $26,006 of gross wages. EITC phaseout uses the stated earned income, which fully eliminates the childless credit." +us,scenario_077,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model incorrectly reduced EITC earned income from $26,006 to $17,617 by subtracting the employer-sponsored insurance premium. At $26,006, the childless EITC is fully phased out rather than reduced to $29." +us,scenario_077,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented a qualifying child from the benchmark’s requested school-meal outputs even though the household lists only one 37-year-old adult and unlisted household facts are false. With no qualifying child, no refundable Child Tax Credit is available." +us,scenario_077,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model explicitly derived zero childless EITC at $26,006 of earned income and identified no other refundable credit, but submitted $111. It failed to carry its own zero-total derivation into the numeric output." +us,scenario_077,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model subtracted the $8,389 insurance premium from wages to produce $17,617 of EITC earned income and then subtracted the capital-loss deduction again to produce $14,617 of AGI. The phaseout must use the greater of the actual $26,006 earned income and AGI, which fully eliminates the childless EITC." +us,scenario_077,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric federal_refundable_credits output or explanation, violating the required output contract." +us,scenario_077,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated $26,006 of childless earnings as producing a phased-in $1,478 credit. For a taxpayer without qualifying children, the credit reaches a much smaller maximum and is fully phased out before this income level." +us,scenario_077,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model assigned a $5,034 EITC at $26,006 of earnings, an amount associated with schedules for taxpayers with qualifying children rather than the childless schedule. With no qualifying children, the lower childless EITC is fully phased out at this income." +us,scenario_077,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model computed about $23,006 of MAGI and then treated that amount as at or below thresholds it stated were approximately $21,000–$22,000. That comparison is arithmetically false; the engine’s MAGI ratio is 1.44 FPL, above the expansion limit, leaving the person with no Medicaid category." +us,scenario_077,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the entire $8,389 employer-sponsored insurance premium from wages without a fact establishing that the employee paid it through a pre-tax salary reduction. PolicyEngine computes MAGI at 1.44 FPL, which does not satisfy Louisiana’s adult expansion pathway." +us,scenario_077,head_medicaid_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by both the full $8,389 employer-sponsored insurance premium and a $3,000 capital-loss deduction to produce $14,617 of MAGI. PolicyEngine instead computes Medicaid MAGI at 1.44 FPL, so the head qualifies through no Medicaid pathway." +us,scenario_077,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The stated $14,617 AGI embeds unsupported deductions for the employer-sponsored insurance premium and capital loss. The Medicaid MAGI computation yields 1.44 FPL, above Louisiana’s expansion limit, and no other eligibility category applies." +us,scenario_077,head_medicaid_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an understated $14,617 MAGI produced by subtracting amounts that the Medicaid MAGI calculation does not deduct as assumed. The engine’s MAGI level is 1.44 FPL, so the head receives category NONE rather than expansion-adult eligibility." +us,scenario_077,head_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The answer asserts that income is below Louisiana’s adult Medicaid threshold, but the applicable computation places MAGI at 1.44 FPL. That exceeds the expansion limit, and the head qualifies through no alternative Medicaid category." +us,scenario_077,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model’s $14,617 MAGI is understated by unsupported subtraction of the employer-sponsored insurance premium and capital-loss amount. PolicyEngine computes 1.44 FPL, which fails Louisiana’s expansion pathway and leaves the head without another qualifying category." +us,scenario_077,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for head_medicaid_eligible. It therefore failed the required output contract rather than completing the Medicaid eligibility calculation. +us,scenario_077,head_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model declared that the head’s estimated income falls below Louisiana’s expansion threshold without performing the applicable MAGI comparison. The engine computes MAGI at 1.44 FPL, above the limit, and assigns Medicaid category NONE." us,scenario_077,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_077,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"This model subtracted the $8,389 employer-sponsored insurance premium from wages before applying FICA, reducing the payroll tax base to $17,617. PolicyEngine applied employee Social Security and Medicare taxes to the wage base used in the trace, yielding $1,612.38 of Social Security tax and $377.09 of Medicare tax." -us,scenario_077,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"This model used the correct employee FICA structure on gross wages but rounded or computed the final amount incorrectly. Applying 6.2% Social Security and 1.45% Medicare to the $26,006 wage base produces about $1,989.47, not $1,988.00." -us,scenario_077,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"This model treated employer-sponsored insurance premiums as reducing FICA-taxable wages and applied 7.65% to $17,617 instead of the wage base used for employee payroll tax. That omitted the payroll tax attributable to the $8,389 of wages it removed from the base." -us,scenario_077,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"This model incorrectly deducted the $8,389 employer-sponsored insurance premium from gross wages when computing employee payroll tax. The benchmark computes employee Social Security and Medicare tax from the wage base in the trace, so the omitted $8,389 of wages accounts for the shortfall." -us,scenario_077,payroll_tax,gpt-5.4-mini,llm_error,other,False,"This model stated the correct employee-side rates but produced a total consistent with roughly doubling employee FICA by including employer-side payroll tax or otherwise applying about 15.26% to wages. The requested output excludes employer payroll taxes and includes only employee Social Security, employee Medicare, Additional Medicare Tax, and mandatory employee state payroll taxes." -us,scenario_077,payroll_tax,gpt-5.4-nano,llm_error,other,False,"This model named the correct wage base and employee FICA rates but made a basic arithmetic error. The calculation it wrote, $26,006 times 6.2% plus $26,006 times 1.45%, equals about $1,989.47, not $1,120.68." -us,scenario_077,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"This model subtracted the $8,389 employer-sponsored insurance premium from gross wages and computed FICA on $17,617. PolicyEngine did not reduce the employee payroll tax wage base by that premium, so the model left out Social Security and Medicare tax on the removed wages." -us,scenario_077,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"This model did not provide a parseable payroll_tax value or supporting explanation. The required computation sums employee Social Security tax of $1,612.38 and employee Medicare tax of $377.09 for a payroll tax value of $1,989.47." +us,scenario_077,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the entire $8,389 employer-sponsored insurance premium as an employee pre-tax payroll deduction and reduced FICA wages to $17,617. The prompt does not identify that premium as an employee pre-tax contribution, so Social Security and Medicare apply to the full $26,006 of wages." +us,scenario_077,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model identified the correct 6.2% Social Security and 1.45% Medicare rates and the correct $26,006 wage base but rounded or computed their product incorrectly. The two components total $1,989.47, not $1,988." +us,scenario_077,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly deducted all $8,389 of employer-sponsored insurance premiums from FICA wages. Because no employee pre-tax payroll contribution is listed, the taxable wage base remains $26,006 rather than $17,617." +us,scenario_077,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model assumed the listed employer-sponsored insurance premiums were employee-paid through a pre-tax cafeteria plan and subtracted them from wages. That unsupported deduction reduced the FICA base from $26,006 to $17,617 and understated both payroll-tax components." +us,scenario_077,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The submitted $3,969 is approximately twice the employee-side FICA liability, so the model effectively included the employer share or doubled the employee tax despite stating that it calculated only employee Social Security and Medicare. Employee-side tax on $26,006 is $1,989.47." +us,scenario_077,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model wrote the correct formula but evaluated it incorrectly: $26,006 × 0.062 plus $26,006 × 0.0145 equals $1,989.47, not $1,120.68. Its error is arithmetic, not a rate or eligibility issue." +us,scenario_077,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly classified the full $8,389 employer-sponsored insurance premium as an employee pre-tax deduction from FICA wages. With no such employee contribution specified, both Social Security and Medicare taxes use the full $26,006 wage base." +us,scenario_077,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." us,scenario_077,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_077,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model recognized that monthly wages exceed the ordinary 130% gross limit but then incorrectly treated Louisiana broad-based categorical eligibility as waiving the decisive income screen and allowing a positive allotment. It also used a mortgage balance as if it were current shelter expense and applied medical deductions to a nonelderly, nondisabled adult, neither of which makes this household SNAP-eligible." -us,scenario_077,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced SNAP countable earned income with capital losses, treating an income-tax AGI concept as if it controlled SNAP eligibility. It also acknowledged that gross wages exceed the SNAP gross limit and net income exceeds the net limit, then contradicted that calculation by assigning a positive benefit instead of stopping at ineligibility." -us,scenario_077,snap,grok-4.3,llm_error,thresholds_rates,False,"The model skipped the SNAP gross and net income screens and classified a single adult earning $26,006 annually as income-eligible. Applying the Louisiana SNAP eligibility thresholds to about $2,167 of monthly wages yields no eligibility and therefore no annual benefit." -us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 Louisiana standard deduction instead of $12,835, then discarded its own $315.18 calculation and introduced an unsupported adjustment to $496. The required computation is $10,171.18 of taxable income at 3%, yielding $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model falsely treated Louisiana as having no individual income tax. Louisiana imposes a 3% tax in 2026, and $10,171.18 of taxable income produces $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 deduction instead of Louisiana's $12,835 single standard deduction. That overstated taxable income by $335 and tax by $10.05." -us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model applied a $12,500 combined exemption or deduction instead of the $12,835 Louisiana standard deduction. Correct taxable income is $10,171.18, not $10,506." -us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model first computed approximately $315 from a $12,500 deduction, then replaced that result with $446 based on a nonexistent rounding or capital-loss adjustment. Louisiana uses $23,006.18 of AGI, a $12,835 deduction, and a 3% rate, yielding $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete $4,500 personal exemption and a federal-income-tax deduction instead of Louisiana's $12,835 standard deduction. Those incorrect deductions produced $17,705 of taxable income rather than $10,171.18." -us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model identified a calculation near $315 using the wrong $12,500 deduction, then invented unspecified federal-based adjustments that raised the answer to $448. No such upward adjustment applies; the $12,835 deduction and 3% rate yield $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 standard deduction instead of $12,835. This made taxable income $10,506 rather than $10,171.18 and tax $315.18 rather than $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted the $8,389 employer-sponsored insurance premium from the stated gross wages to derive $14,617 of AGI, then used an obsolete $4,500 deduction. AGI is $23,006.18 after the federal $3,000 capital-loss limit, and Louisiana subtracts $12,835." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $158 answer does not result from Louisiana's 2026 computation. Taxable income is $10,171.18 and the applicable flat rate is 3%, producing $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly excluded the $8,389 insurance premium from wages and double-counted separate $4,500 deductions. Louisiana instead starts from $23,006.18 of AGI and subtracts one $12,835 standard deduction." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced AGI to $14,617 by subtracting employer-sponsored insurance premiums and then applied an obsolete $4,500 deduction. The correct AGI is $23,006.18 and the correct Louisiana deduction is $12,835." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer implies the model used the common but wrong $12,500 deduction: $315.18 divided by 3% equals $10,506 of taxable income. Louisiana's $12,835 deduction leaves $10,171.18 and produces $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 Louisiana standard deduction rather than $12,835. The resulting $335 taxable-income overstatement increased tax from $305.14 to $315.18." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model invented a zero-tax threshold above this household's taxable income. After the $12,835 deduction, $10,171.18 remains taxable at 3%, so the liability is positive." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model treated low wages and the absence of listed credits as grounds for zero liability without calculating Louisiana taxable income. The $12,835 deduction leaves $10,171.18 subject to the 3% tax." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 standard deduction instead of $12,835. Correcting the deduction reduces taxable income from $10,506 to $10,171.18 and tax from $315.18 to $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model estimated the Louisiana single deduction as $12,500 rather than applying $12,835. Its rate was correct, but its taxable-income base was $335 too high." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted $12,500 instead of the $12,835 Louisiana standard deduction. The correct base for the 3% rate is $10,171.18." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used the wrong $12,500 standard deduction and additionally subtracted a nonexistent separate $4,500 personal exemption. Louisiana subtracts the $12,835 standard deduction without that extra exemption." -us,scenario_077,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly concluded that deductions eliminate all Louisiana taxable income. Federal AGI of $23,006.18 less the $12,835 standard deduction leaves $10,171.18 taxable." -us,scenario_077,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied an obsolete $4,500 exemption and graduated 1.85% and 3.5% rates. Louisiana's 2026 calculation uses a $12,835 standard deduction and a flat 3% rate." -us,scenario_077,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted employer-sponsored insurance premiums from the given wages to obtain $14,617 of AGI and used an obsolete $4,500 deduction. The correct inputs are $23,006.18 of AGI and a $12,835 Louisiana standard deduction." -us,scenario_077,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required structured-output contract before any substantive tax calculation could be evaluated. -us,scenario_077,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 standard deduction rather than Louisiana's $12,835 deduction. This overstated taxable income by $335 and produced $315.18 instead of $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model falsely stated that Louisiana has no state income tax. Louisiana's 2026 flat 3% tax applies to $10,171.18 of taxable income." -us,scenario_077,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model deducted the full $7,968 capital loss instead of applying the $3,000 federal AGI limit, used an obsolete $4,500 exemption, and applied obsolete graduated rates. Louisiana starts from $23,006.18 of AGI, subtracts $12,835, and applies a flat 3% rate." -us,scenario_077,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model invented a 2026 Louisiana refundable EITC equal to 5% of federal EITC and placed it in state_refundable_credits. The Louisiana refundable credit computation used for this household contributes $0, so multiplying an estimated federal EITC by 5% is the wrong state-law step." -us,scenario_077,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model treated Louisiana as offering a refundable EITC equal to 5% of federal EITC in 2026, then rounded 5% of its own $108 federal EITC estimate to $5. The PolicyEngine Louisiana refundable credits path for this household returns $0, so the model added a state refundable credit that is not available in the reference computation." -us,scenario_077,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model applied a Louisiana refundable earned income credit at 5% of an estimated federal EITC and therefore converted a federal EITC estimate into $31.25 of state refundable credits. The correct state computation for Louisiana in the reference contributes no refundable credit for this household, making the 5% Louisiana EITC shortcut the error." -us,scenario_077,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable state_refundable_credits value or supporting explanation. This is a missing-output failure rather than a substantive tax-rule computation. +us,scenario_077,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly applied a 200% FPL broad-based categorical-eligibility pathway and then asserted that categorical eligibility waived the net-income test. It also converted the reported mortgage balance into unspecified shelter costs even though a loan balance is not a SNAP shelter expense and all unlisted mortgage payments, taxes, insurance, and utilities are zero." +us,scenario_077,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly used the household's capital losses to reduce SNAP countable earned income. SNAP counts the $26,006 of wages under its own income rules rather than federal AGI, so those investment losses do not cure the failed gross-income test; its own stated net income of about $1,530 also exceeds the net limit it cited." +us,scenario_077,snap,grok-4.3,llm_error,thresholds_rates,False,"The model labeled the household low-income without applying the one-person gross-income threshold. Monthly wages of about $2,167 exceed the applicable Louisiana SNAP gross-income limit, so the household does not reach the allotment calculation and receives $0." +us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 Louisiana standard deduction instead of $12,835, then abandoned its own $315.18 computation and asserted $496 without a valid adjustment. The correct taxable income is $10,171.18 and the resulting tax is $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model falsely treated Louisiana as a state without an individual income tax. Louisiana taxes this filer’s $10,171.18 of taxable income, producing $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model substituted a $12,500 deduction for Louisiana’s $12,835 standard deduction. That overstated taxable income by $335 and produced $315 instead of $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 combined exemption or deduction instead of the applicable $12,835 Louisiana standard deduction. Louisiana taxable income is therefore $10,171.18, not $10,506." +us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly approached a roughly $315 calculation using its assumed deduction, then replaced it with $446 based on nonexistent PolicyEngine rounding and capital-loss adjustments. PolicyEngine’s AGI is $23,006.18, its Louisiana deduction is $12,835, and the tax is $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete $4,500 personal exemption and a federal-income-tax deduction instead of Louisiana’s $12,835 standard deduction. Those errors inflated taxable income to $17,705 rather than $10,171.18." +us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model’s stated 3% calculation produced about $315, but it then asserted $448 through unspecified federal-based adjustments and personal-exemption credits. No such step produces the requested liability; the traced deduction and tax computation yield $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 standard deduction rather than $12,835. This left taxable income at $10,506 instead of $10,171.18 and overstated the tax." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the $8,389 employer-sponsored insurance premium from the stated gross wages when constructing AGI, reducing it to $14,617. It also applied an obsolete $4,500 deduction and 1.85% bracket instead of the 2026 Louisiana deduction and tax schedule." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The unsupported $158 answer omits the traced computation from $23,006.18 of AGI through the $12,835 Louisiana standard deduction. That derivation leaves $10,171.18 taxable and produces $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced AGI to $14,617 by treating employer-sponsored insurance premiums as an additional wage deduction, then double-counted separate $4,500 deductions. It also used an obsolete 1.85% rate rather than the applicable 2026 calculation." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the employer-sponsored insurance premiums from wages to obtain $14,617 of AGI. It then used an obsolete $4,500 deduction and 1.85% bracket instead of the $12,835 deduction and applicable 2026 tax schedule." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer implies a $12,500 deduction because $315.18 equals 3% of $10,506. Louisiana’s applicable deduction is $12,835, leaving $10,171.18 taxable and $305.14 of tax." +us,scenario_077,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model substituted a $12,500 Louisiana standard deduction for the applicable $12,835 amount. Its taxable-income base was consequently $335 too high." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model incorrectly treated the remaining taxable income as below a zero-tax threshold. After the $12,835 standard deduction, $10,171.18 remains taxable and generates $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model assumed low wages and unspecified credits eliminated the liability without calculating Louisiana taxable income. The requested measure precedes refundable credits, and $10,171.18 of taxable income produces $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 standard deduction rather than Louisiana’s $12,835 deduction. This overstated taxable income from $10,171.18 to $10,506 and tax from $305.14 to $315.18." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model estimated the Louisiana standard deduction as $12,500 instead of applying $12,835. The correct taxable-income base is $10,171.18, not $10,506." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used the wrong Louisiana standard deduction, $12,500 rather than $12,835. Correcting that input reduces taxable income to $10,171.18 and tax to $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model deducted both a $12,500 standard deduction and a separate $4,500 personal exemption. Louisiana’s traced computation instead subtracts a single $12,835 standard deduction from AGI, leaving $10,171.18 taxable." +us,scenario_077,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly concluded that Louisiana deductions eliminate all taxable income. AGI of $23,006.18 less the $12,835 standard deduction leaves positive taxable income of $10,171.18." +us,scenario_077,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied an obsolete $4,500 exemption and graduated 1.85% and 3.5% brackets. The 2026 computation uses the $12,835 standard deduction and yields $305.14 on $10,171.18 of taxable income." +us,scenario_077,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted employer-sponsored insurance premiums from the stated wages to reduce AGI to $14,617. It also used an obsolete $4,500 deduction and 1.85% rate instead of the applicable 2026 Louisiana rules." +us,scenario_077,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 single-filer deduction rather than $12,835. That shortcut produced approximately $315 instead of the traced $305.14 liability." +us,scenario_077,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the submission contract." +us,scenario_077,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model applied a $12,500 standard deduction instead of Louisiana’s $12,835 amount. This overstated taxable income by $335 and produced $315.18 rather than $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely stated that Louisiana has no state income tax. Louisiana imposes tax on this filer’s $10,171.18 of taxable income, resulting in $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model deducted the entire $7,968 net capital loss instead of applying the $3,000 annual capital-loss limit, reducing AGI to $18,038 rather than $23,006.18. It then used obsolete graduated rates and a $4,500 exemption instead of the $12,835 standard deduction and applicable 2026 tax calculation." +us,scenario_077,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model gave $347.24 without specifying the deduction amount, taxable-income base, or rate computation that generated it. The traced calculation subtracts $12,835 from $23,006.18 and taxes the resulting $10,171.18 to produce $305.14." +us,scenario_077,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model applied Louisiana’s former 5% refundable EITC in tax year 2026. That state credit is unavailable for 2026, so multiplying an estimated federal EITC by 5% incorrectly produced $10.53 instead of zero." +us,scenario_077,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied Louisiana’s former refundable EITC rate of 5% to a purported $108 federal EITC. Louisiana provides no refundable EITC for tax year 2026, so the state refundable credit is zero." +us,scenario_077,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model incorrectly carried Louisiana’s former 5% refundable EITC into 2026 and also assigned a $625 federal EITC to a childless filer with $26,006 of earnings, above the childless-credit phaseout range. The unavailable state credit cannot produce the claimed $31.25." +us,scenario_077,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required structured-output contract." us,scenario_077,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $1,632 state and local tax refund in current AGI and deducted only $26,460 of SALT instead of the full $36,233.77 deduction. It also summed its own bracket components incorrectly: the stated components total $27,510.08, not $33,499." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $1,632 refund as taxable and the suspended $2,606 unreimbursed employee expense as an above-the-line deduction. It also omitted $9,773.77 of deductible SALT, so it used $142,861 rather than $134,061.22 of taxable income." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete $10,000 SALT cap instead of the applicable 2026 SALT deduction of $36,233.77 and incorrectly added the $1,632 refund to AGI. Those errors raised taxable income from $134,061.22 to $161,927." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model applied a $10,000 SALT cap and included the $1,632 refund in AGI, producing $161,927 instead of $134,061.22 of taxable income. It then abandoned its own $31,424 bracket calculation and submitted the unsupported $36,757 figure." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model capped SALT at $10,000 instead of deducting $36,233.77 and incorrectly included the $1,632 refund in AGI. It therefore taxed $161,927 rather than $134,061.22." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used the obsolete $10,000 SALT cap and treated the $1,632 refund as taxable current-year income. This understated itemized deductions by $26,233.77 and overstated taxable income by $27,865.77." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model limited SALT to $10,000 instead of $36,233.77, leaving taxable income far too high. It then replaced its own $31,069 income-tax computation with $42,800 by improperly invoking Medicare-tax interactions, which do not belong in this output." -us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly applied pre-TCJA rates and a personal exemption rather than the applicable 2026 bracket and deduction rules. It also included the refund in AGI and used estimated deductions instead of the traced $64,443.77, missing taxable income of $134,061.22." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset, restored a personal exemption, and applied the old 10/15/25/28 percent rate schedule. The applicable computation uses no personal exemption and applies the 2026 brackets to $134,061.22." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model took only a $15,000 standard deduction and ignored the larger $64,443.77 itemized deduction. That shortcut taxed $183,505 instead of $134,061.22." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model wrongly restored post-sunset personal exemptions and invoked AMT. The regular 2026 computation uses $64,443.77 of itemized deductions, $134,061.22 of taxable income, and no AMT increase." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset and deducted a nonexistent $5,150 personal exemption. It also estimated the SALT component rather than using $36,233.77, so it failed to apply the current 2026 brackets to $134,061.22." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly deducted a $5,050 personal exemption and included the $1,632 refund in AGI. Those choices produced $128,877.94 rather than the traced $134,061.22 of taxable income and led it to the wrong bracket result." -us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model ignored the $64,443.77 itemized deduction and fabricated a $12,000 child/dependent-care credit despite there being no dependent or qualifying care expense. CDCC is unavailable, and the correct liability comes from applying the brackets to $134,061.22 with no nonrefundable credit reduction." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model applied a $10,000 SALT cap instead of deducting $36,233.77 of SALT. Its $42,034 answer also exceeds the liability generated by the taxable-income calculation it described, so it did not correctly apply the progressive brackets." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The submitted amount is inconsistent with the traced deduction computation: mortgage interest plus SALT totals $64,443.77 and leaves $134,061.22 taxable. Applying the 2026 brackets to that amount yields $24,772.69, not $28,347." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $1,632 refund in AGI and omitted $9,773.77 of deductible SALT, producing $145,467 instead of $134,061.22 of taxable income. It also misapplied the 2026 brackets to its own taxable-income figure." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model did not use the full traced itemized deduction of $64,443.77. The mortgage-interest and SALT deductions leave $134,061.22 taxable, whose 2026 bracket liability is $24,772.69 rather than $25,947.40." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $1,632 refund to AGI and limited itemized deductions to mortgage interest plus property tax, omitting $9,773.77 of additional deductible SALT. It therefore used $145,467 rather than $134,061.22 of taxable income." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model deducted only mortgage interest and real-estate tax, omitting the rest of the $36,233.77 SALT deduction. The complete $64,443.77 itemized deduction produces $134,061.22 of taxable income and $24,772.69 of tax." -us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $41,500 answer is inconsistent with the household's $64,443.77 itemized deduction. Deducting that amount from $198,505 leaves $134,061.22 taxable, and the 2026 brackets produce $24,772.69." -us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly included the refund in AGI, restored a $5,427 personal exemption, and applied a pre-TCJA rate schedule. The applicable computation has no personal exemption and applies the 2026 brackets to $134,061.22." -us,scenario_078,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly included the refund in AGI, deducted a restored personal exemption, and used pre-TCJA brackets. The correct 2026 computation uses $64,443.77 of itemized deductions, no personal exemption, and $134,061.22 of taxable income." -us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly recognized the higher 2026 SALT cap but counted only the listed $26,460 property tax, omitting $9,773.77 of additional deductible SALT, and it incorrectly included the $1,632 refund in AGI. Those errors produced $145,467 instead of $134,061.22 of taxable income." -us,scenario_078,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model treated the filer as having no federal income-tax liability despite $198,505 of wages. After $64,443.77 of itemized deductions, $134,061.22 remains taxable and generates $24,772.69 before refundable credits." -us,scenario_078,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete $10,000 SALT cap and included the $1,632 refund in AGI, yielding $161,927 instead of $134,061.22 of taxable income. It also used 2024 brackets after repeatedly changing its projected 2026 thresholds." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $1,632 state and local tax refund to current-year AGI and limited SALT to the listed $26,460 real estate tax. The trace instead uses AGI of $198,505 and a $36,233.77 SALT deduction, leaving taxable income of $134,061.22; it also arithmetically summed its bracket components incorrectly, since its stated components total $27,510.08 rather than $33,499." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly treated the refund as taxable income, deducted unreimbursed employee business expenses from AGI, and counted only $26,460 of SALT. Unreimbursed employee expenses do not reduce this AGI, and the trace uses $36,233.77 of SALT and total itemized deductions of $64,443.77." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete $10,000 SALT cap and added the $1,632 refund to AGI. The applicable 2026 calculation deducts $36,233.77 of SALT, excludes the refund from AGI, and produces taxable income of $134,061.22 rather than $161,927." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model applied a $10,000 SALT cap and included the $1,632 refund in AGI instead of using the trace's $36,233.77 SALT deduction and $198,505 AGI. Its submitted $36,757 also contradicts its own bracket calculation of approximately $31,424." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used the obsolete $10,000 SALT cap and taxed the $1,632 refund. This understated itemized deductions by $26,233.77 and overstated AGI by $1,632, producing $161,927 of taxable income instead of $134,061.22." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model capped SALT at $10,000 and included the refund in gross income. The 2026 trace deducts $36,233.77 of SALT and starts from $198,505 of AGI, so its taxable-income base was overstated by $27,865.77." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model applied a $10,000 SALT cap rather than the $36,233.77 deduction used in 2026. It then abandoned its own regular-income-tax calculation near $31,069 and inflated the answer with Medicare-tax interactions, even though payroll and Additional Medicare taxes are excluded from this output." +us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model treated 2026 as a reversion to pre-TCJA brackets and restored a personal exemption. The applicable calculation uses the 2026 10%, 12%, 22%, and 24% bracket schedule with no personal exemption, applied to $134,061.22 of taxable income." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration restored a personal exemption and the pre-TCJA 10%, 15%, 25%, and 28% brackets. The trace uses no personal exemption and applies the operative 2026 brackets to taxable income of $134,061.22." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used the standard deduction and ignored $64,443.77 of qualifying itemized deductions. Itemizing the $28,210 mortgage interest and $36,233.77 SALT deduction reduces taxable income to $134,061.22." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly restored personal exemptions and postulated an AMT liability under a TCJA-sunset regime. The operative 2026 regular-tax calculation uses $64,443.77 of itemized deductions and applies the regular brackets to $134,061.22 without the asserted AMT increase." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly restored a $5,150 personal exemption and used a TCJA-sunset tax regime. The trace allows no personal exemption and instead derives $134,061.22 of taxable income from $64,443.77 of itemized deductions under the operative 2026 law." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly added the refund to AGI and restored a $5,050 personal exemption under a presumed TCJA sunset. The trace uses AGI of $198,505, no personal exemption, and $64,443.77 of itemized deductions before applying the operative 2026 brackets." +us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model invented a $12,000 child and dependent care credit despite the household containing no child, dependent, qualifying care expense, or other eligible person. It also ignored the $64,443.77 itemized deduction and therefore began from an overstated taxable-income base." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model capped SALT at $10,000 instead of deducting $36,233.77 and did not carry the qualifying itemized deductions through to the correct taxable income. The proper itemized total is $64,443.77, leaving $134,061.22 subject to the 2026 brackets." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The submitted amount does not reflect the trace's full $36,233.77 SALT deduction and $64,443.77 total itemized deduction. Those deductions reduce taxable income to $134,061.22, whose bracket tax is $24,772.69." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly taxed the $1,632 refund and limited SALT to the $26,460 real estate tax. The trace excludes the refund from AGI and deducts $36,233.77 of SALT, reducing taxable income from the model's $145,467 to $134,061.22." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model failed to use the traced $36,233.77 SALT deduction together with $28,210 of mortgage interest. The resulting itemized deduction is $64,443.77 and taxable income is $134,061.22, not the higher base implicit in $25,947.40." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $1,632 refund to AGI and counted only the $26,460 real estate tax as SALT. The trace instead uses $198,505 of AGI and $36,233.77 of SALT, lowering taxable income from $145,467 to $134,061.22." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model limited the SALT deduction to the listed $26,460 real estate tax and omitted the remainder of the traced $36,233.77 SALT deduction. Total itemized deductions are $64,443.77, producing $134,061.22 of taxable income." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $41,500 answer is inconsistent with itemizing $64,443.77 against $198,505 of AGI. The traced deductions leave $134,061.22 of taxable income, and the submitted number reflects failure to apply that itemized-deduction computation." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly added the refund to AGI, restored a $5,427 personal exemption, and applied a pre-TCJA-style bracket schedule. The operative calculation uses no personal exemption, $198,505 of AGI, $64,443.77 of deductions, and the 2026 brackets on $134,061.22." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly restored a $5,434 personal exemption and applied pre-TCJA brackets. It also taxed the refund and used an independently estimated SALT amount instead of the traced $36,233.77; the operative calculation taxes $134,061.22 under the 2026 bracket schedule." +us,scenario_078,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model taxed the $1,632 refund and limited SALT to the $26,460 real estate tax. The trace uses AGI of $198,505 and a $36,233.77 SALT deduction, which reduces taxable income to $134,061.22." +us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output for federal_income_tax_before_refundable_credits. It therefore failed the required structured-output contract. +us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $1,632 refund in AGI and treated the $26,460 real estate tax as the entire SALT deduction. The trace uses $36,233.77 of SALT and $198,505 of AGI, yielding $134,061.22 of taxable income." +us,scenario_078,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model returned zero despite $198,505 of wage income remaining taxable after itemized deductions. The traced $64,443.77 deduction leaves $134,061.22 of taxable income and $24,772.69 of regular federal income tax." +us,scenario_078,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete $10,000 SALT cap, included the $1,632 refund in AGI, and ultimately used 2024 rather than 2026 brackets. The trace deducts $36,233.77 of SALT from $198,505 of AGI and applies the 2026 brackets to $134,061.22." +us,scenario_078,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly constructed an AMT liability, including a fabricated AMT offset and an unexplained $1,127.51 rounding addition. The traced regular-tax computation uses $64,443.77 of itemized deductions and applies the 2026 brackets to $134,061.22, with no such AMT adjustment." us,scenario_078,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_078,local_income_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model treated Maryland county income tax as the benchmark's `local_income_tax` variable and applied an estimated 3.03% county rate to Maryland taxable income. This output is limited to the separate NYC, Philadelphia, Kansas City, and St. Louis local income, wage, and earnings taxes, so no Maryland local rate applies and the computed liability is $0." -us,scenario_078,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly derived $11,439 of Social Security tax plus $2,878.32 of Medicare tax, totaling $14,317.32, but submitted $12,240.44 instead. Its numeric output discarded its own completed payroll-tax calculation." -us,scenario_078,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model used the obsolete $137,700 Social Security wage base instead of the 2026 base of $184,500, then invented a Maryland employee payroll contribution. Maryland adds no employee payroll tax in this calculation; the federal components total $14,317.32." -us,scenario_078,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model correctly calculated Social Security tax of $11,439 and Medicare tax of $2,878.32, totaling $14,317.32, but submitted $12,550.53. The submitted value does not carry through either component calculation." -us,scenario_078,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model first derived the exact $14,317.32 total, then substituted another incorrect Social Security wage-base estimate and ultimately submitted $11,611.86, which matches neither derivation. It failed to carry its payroll-tax components into the final output." -us,scenario_078,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model capped Social Security wages at $176,100 instead of the 2026 taxable maximum of $184,500. This understated Social Security tax by $520.80 while its Medicare and Additional Medicare Tax treatment was correct." -us,scenario_078,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model used a $184,000 Social Security wage base instead of $184,500, understating Social Security tax by $31. It then rounded away another $0.32 rather than submitting the annual tax amount exactly." -us,scenario_078,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model used an incorrect $177,600 Social Security wage cap and added a Maryland FAMLI employee contribution that does not apply in this calculation. The applicable components are only $11,439 of Social Security tax and $2,878.32 of Medicare tax." -us,scenario_078,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model used the 2024 Social Security wage base of $168,600 rather than the 2026 base of $184,500. Its submitted total also does not equal the Social Security and Medicare amounts stated in its explanation." -us,scenario_078,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model added an estimated Maryland FAMLI payroll contribution even though no mandatory Maryland employee payroll tax enters this case. Applying only the 2026 Social Security wage cap and Medicare tax yields $14,317.32." -us,scenario_078,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model capped Social Security wages at $181,800 instead of $184,500, understating Social Security tax by $167.40. It also rounded its own $14,149.92 component sum to $14,150." -us,scenario_078,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model used a Social Security taxable maximum of $181,800 instead of the 2026 limit of $184,500. That reduced Social Security tax by $167.40; Medicare tax was computed correctly." -us,scenario_078,payroll_tax,glm-5.2,llm_error,other,False,"The model used an incorrect $176,100 Social Security wage base and derived $13,796.52, but then submitted the unrelated value $11,870.18. The final output therefore failed both the wage-base rule and the model's own arithmetic." -us,scenario_078,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The submitted $15,463 implies an excessive Social Security component after the correctly described 1.45% Medicare tax is removed. Social Security is capped at $184,500, so it equals $11,439 and the total equals $14,317.32." -us,scenario_078,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,The model invoked the combined employer-and-employee FICA rates of 12.4% and 2.9% even though the requested output includes only employee-side rates of 6.2% and 1.45%. It also failed to apply those stated rates consistently to its submitted number. -us,scenario_078,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model used $183,900 instead of the $184,500 Social Security wage base and then applied an unexplained reduction tied to Maryland paid leave. No such reduction offsets federal employee FICA, so the correct components sum directly to $14,317.32." -us,scenario_078,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The $11,485 answer omits $2,832.32 of the required employee payroll tax. The derivation requires $11,439 of capped Social Security tax plus $2,878.32 of uncapped Medicare tax, with no Additional Medicare Tax." -us,scenario_078,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model used an approximate $183,700 Social Security wage base instead of the 2026 limit of $184,500. It also rounded component amounts, producing a submitted total $49.32 below the exact calculation." -us,scenario_078,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax value. It therefore failed the required outputs contract before any substantive payroll-tax calculation was presented. -us,scenario_078,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated payroll tax as zero despite $198,505 of employee wages. Those wages generate $11,439 of employee Social Security tax and $2,878.32 of employee Medicare tax." -us,scenario_078,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model projected a $174,900 Social Security wage base instead of applying the 2026 taxable maximum of $184,500. This understated taxable Social Security wages by $9,600 and Social Security tax by $595.20." -us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model selected a Maryland standard deduction instead of the larger $54,670 itemized deduction. That overstated taxable income from $143,835 to roughly $195,005 and consequently overstated the tax." -us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model did not deduct the full $54,670 of Maryland itemized deductions, despite identifying the mortgage interest and property tax. Maryland taxable income is $143,835, not approximately $195,305." -us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly derived $143,835 of taxable income and approximately $6,936.59 of Maryland tax, then submitted $9,755 instead. Its numeric output contradicts its completed bracket calculation." -us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model arbitrarily reduced Maryland itemized deductions to about $30,000 rather than deducting the full $54,670 of mortgage interest and property tax. It therefore used about $168,505 of taxable income instead of $143,835, and its submitted value also contradicts its own $8,277.78 calculation." -us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model reduced the itemized deduction to roughly $38,210 rather than using the full $54,670. It also applied an incorrect Maryland rate schedule, describing a 5.0% top marginal rate where the taxable-income bands reach 5.25%." -us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model chose the Maryland standard deduction and a personal exemption instead of the beneficial $54,670 itemized deduction. This inflated taxable income to $194,537 rather than $143,835." -us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly reached Maryland taxable income of $143,835 after the $54,670 itemized deduction, but then failed to apply the stated progressive rate schedule. That schedule yields $6,936.34, not $9,200." -us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $2,400 standard deduction and $3,200 personal exemption instead of the larger $54,670 itemized deduction. This produced $194,537 of taxable income rather than $143,835." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies taxation of nearly the full Maryland AGI rather than taxable income of $143,835. It omitted the beneficial $54,670 itemized deduction for mortgage interest and property tax." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model included a typical Maryland county tax even though the requested state-income-tax output excludes local income taxes. The state-only liability after the $54,670 itemized deduction is $6,936.34." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly calculated taxable income of $143,835 but misapplied Maryland's graduated rate schedule. Applying the schedule to that base yields $6,936.34, leaving its estimate $50.34 too low." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model explicitly added $4,602.72 of county tax to the correctly calculated $6,936.34 state tax. The requested output excludes local income taxes, so only $6,936.34 belongs in this field." -us,scenario_078,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model failed to remove the $1,632 state and local tax refund from Maryland income, using $200,137 rather than $198,505 before deductions. It then also misapplied the bracket schedule to its stated taxable income, producing $7,300.69 instead of the state tax derived from $143,835." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The submitted amount is consistent with taxing nearly the full Maryland AGI rather than subtracting the $54,670 itemized deduction. The correct taxable-income base is $143,835." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Despite referring to itemized deductions, the model's $10,120 estimate reflects no effective subtraction of the full $54,670 deduction. Mortgage interest and property tax reduce Maryland taxable income to $143,835." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model stated that it used allowable itemized deductions but returned tax below the amount produced by Maryland's rate schedule on $143,835. The schedule yields $6,936.34, so its bracket computation was understated." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model recognized the available itemized deductions but did not correctly apply Maryland's graduated rates to the resulting $143,835 taxable income. That calculation yields $6,936.34 rather than $7,025." -us,scenario_078,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $9,200 answer is consistent with taxing a base close to gross income rather than the $143,835 base produced after the $54,670 itemized deduction. The model omitted the decisive Maryland deduction calculation." -us,scenario_078,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used the $2,400 standard deduction and ignored the larger $54,670 itemized deduction. This overstated taxable income to $197,737 instead of $143,835." -us,scenario_078,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value for the requested output. It therefore failed the required output contract before any substantive tax calculation could be evaluated. -us,scenario_078,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model returned zero despite $143,835 of Maryland taxable income. Applying Maryland's graduated rate schedule to that positive base produces $6,936.34 before refundable credits." -us,scenario_078,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model expressly assumed no Maryland-specific deductions and used a standard deduction, ignoring the listed $28,210 mortgage interest and $26,460 property tax. Those itemized deductions total $54,670 and reduce taxable income to $143,835; its final $10,933 also contradicts its own $9,927.78 pre-deduction bracket sum." +us,scenario_078,local_income_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model incorrectly treated Maryland county income tax as part of the requested local_income_tax variable and applied an assumed 3.03% average county rate to taxable income. This output excludes Maryland county tax and only captures the specifically listed modeled city taxes, none of which applies to this household." +us,scenario_078,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly derived $11,439 of Social Security tax plus $2,878.32 of Medicare tax, totaling $14,317.32, but submitted $12,240.44 instead. Its final numeric output contradicts its own completed calculation." +us,scenario_078,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model used the obsolete $137,700 Social Security wage base instead of the 2026 base of $184,500, then invented a Maryland employee payroll contribution to alter the result. Maryland adds no mandatory employee payroll tax here, so the federal components alone total $14,317.32." +us,scenario_078,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model correctly calculated $11,439 of Social Security tax and $2,878.32 of Medicare tax, with no Additional Medicare Tax or Maryland contribution, but submitted $12,550.53. That output discards its own $14,317.32 derivation." +us,scenario_078,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model first derived the exact $14,317.32 total, then introduced an incorrect alternative Social Security wage base and ultimately submitted $11,611.86, which matches neither calculation. The required base is $184,500, yielding $11,439 of Social Security tax." +us,scenario_078,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model capped Social Security wages at $176,100 instead of the 2026 wage base of $184,500. This understated Social Security tax by $520.80; Medicare tax was correctly calculated and no Additional Medicare Tax or Maryland payroll tax applies." +us,scenario_078,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model used a $184,000 Social Security wage base instead of $184,500, understating Social Security tax by $31. It also rounded the resulting total to whole dollars rather than retaining the annual amount of $14,317.32." +us,scenario_078,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model used an incorrect $177,600 Social Security wage base and added a Maryland FAMLI employee contribution that does not apply in this calculation. The applicable components are only $11,439 of Social Security tax and $2,878.32 of Medicare tax." +us,scenario_078,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model used the 2024 Social Security wage base of $168,600 rather than the 2026 base of $184,500. Its stated components also total about $13,331, not the submitted $14,193, creating an additional arithmetic-output inconsistency." +us,scenario_078,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model added an estimated Maryland FAMLI payroll contribution even though no mandatory Maryland employee payroll tax applies here. Social Security at the $184,500 base plus Medicare on all wages totals $14,317.32." +us,scenario_078,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model capped Social Security wages at $181,800 instead of the 2026 base of $184,500. That understated Social Security tax by $167.40; the exact total is $14,317.32 rather than the rounded $14,150." +us,scenario_078,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model used an incorrect $181,800 Social Security wage base. Applying 6.2% to the correct $184,500 base raises Social Security tax from $11,271.60 to $11,439 and produces the $14,317.32 total." +us,scenario_078,payroll_tax,glm-5.2,llm_error,other,False,"The model used an incorrect $176,100 Social Security wage base in its reasoning, which would yield $13,796.52, but then submitted the unrelated value $11,870.18. Using the 2026 base of $184,500 produces $14,317.32." +us,scenario_078,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The submitted $15,463 cannot result from the stated federal rules: Social Security is capped at $184,500 and Medicare applies at 1.45% to $198,505, with no Additional Medicare Tax or Maryland contribution. The model applied an unstated incorrect base, rate, or arithmetic step instead of totaling $14,317.32." +us,scenario_078,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model cited the combined employer-and-employee rates of 12.4% for Social Security and 2.9% for Medicare even though the requested output includes only the employee side. It also produced a number inconsistent with those stated rates; the employee rates are 6.2% and 1.45%, yielding $14,317.32." +us,scenario_078,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model used an incorrect $183,900 Social Security wage base and then applied an unexplained reduction associated with Maryland paid leave. No such reduction belongs in payroll tax, and the correct $184,500 base plus full Medicare tax yields $14,317.32." +us,scenario_078,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The $11,485 answer omits part of the required employee payroll tax. Social Security alone is $11,439 and Medicare adds $2,878.32, so including both required components yields $14,317.32." +us,scenario_078,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model used an approximate $183,700 Social Security wage base instead of the 2026 base of $184,500. This understated the Social Security component and produced $14,268 rather than $14,317.32." +us,scenario_078,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax value. It therefore failed the required output contract rather than completing the calculation. +us,scenario_078,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model returned zero despite $198,505 of employee wages being subject to Social Security and Medicare taxes. The wages generate $11,439 of Social Security tax and $2,878.32 of Medicare tax, totaling $14,317.32." +us,scenario_078,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model projected a $174,900 Social Security wage base instead of using the 2026 base of $184,500. This understated Social Security tax by $595.20 while correctly applying Medicare and the Additional Medicare Tax threshold." +us,scenario_078,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated $10,453.20 of Social Security tax and $2,878.32 of Medicare tax, which sum to $13,331.52, but submitted $15,622.81. It also used the wrong Social Security wage base; the correct Social Security amount is $11,439 and the correct total is $14,317.32." +us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used a standard deduction and an exemption instead of the larger $54,670 Maryland itemized deduction. That left taxable income near $195,005 rather than $143,835 and overstated the tax." +us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invoked the federal SALT cap and described Maryland taxable income near $195,305 instead of deducting the full $28,210 mortgage interest and $26,460 property taxes allowed in the traced Maryland calculation. The correct $54,670 itemized deduction leaves $143,835 taxable." +us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly derived taxable income of $143,835 and tax of approximately $6,936.59, then submitted $9,755 instead. Its final value discarded its own Maryland bracket calculation." +us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model reduced the itemized deduction to an unsupported approximation of $30,000 instead of using $54,670, omitting much of the $26,460 property-tax deduction. It therefore taxed about $168,505 rather than $143,835 and then submitted a value inconsistent even with its stated $8,277.78 calculation." +us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model reduced the Maryland itemized deduction to about $38,210 instead of $54,670, effectively excluding $16,460 of the property taxes. It consequently used taxable income near $161,900 rather than $143,835." +us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model chose Maryland’s small standard deduction and an unphased personal exemption instead of the beneficial $54,670 itemized deduction. It also retained the $1,632 state-tax refund in the Maryland starting income, producing $194,537 of taxable income instead of $143,835." +us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model reached the correct Maryland taxable income of $143,835 after the $54,670 itemized deduction but failed to apply the stated graduated brackets to that base. Those brackets yield $6,936.34, not the submitted $9,200." +us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $2,400 standard deduction and $3,200 personal exemption while ignoring the larger $54,670 itemized deduction. It also included the $1,632 refund in Maryland AGI, leaving taxable income at $194,537 instead of $143,835." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $9,070 reflects taxation of a base far above the traced $143,835 taxable income. The model’s generic federal-AGI shortcut omitted the beneficial $54,670 Maryland itemized deduction." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model explicitly included a typical Maryland county or local income tax in an output that excludes local income tax. Only the Maryland state schedule applies here, producing $6,936.34 on $143,835 of taxable income." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly identified $143,835 of taxable income but misapplied Maryland’s graduated rate schedule. Applying the schedule to that base yields $6,936.34, not $6,886." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model calculated the $6,936.34 state tax correctly and then added $4,602.72 of county tax. County tax belongs outside the requested state-income-tax output, so the submitted total improperly combines state and local liabilities." +us,scenario_078,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model started from $200,137 rather than Maryland adjusted gross income of $198,505, leaving $145,467 after itemization instead of $143,835. It also miscomputed the bracket tax on its stated base, since the extra $1,632 of taxable income cannot explain its $364.35 overstatement." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The submitted $9,720 reflects taxation of income near the pre-itemization base rather than $143,835. The model failed to apply the $54,670 Maryland itemized deduction comprising mortgage interest and property taxes." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Despite mentioning itemized deductions, the submitted $10,120 reflects a taxable base near gross income rather than $143,835. The calculation failed to subtract the full $54,670 Maryland itemized deduction before applying the state brackets." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model stated that it used allowable itemized deductions, but its $6,511.18 result understates the tax produced by Maryland’s schedule on $143,835. The correct graduated-bracket computation yields $6,936.34, with no additional personal exemption in the traced taxable-income figure." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified itemization and no state credits but did not compute Maryland’s graduated schedule accurately. Tax on the resulting $143,835 taxable income is $6,936.34 rather than the rounded $7,025 estimate." +us,scenario_078,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $9,200 answer is consistent with taxing a base near adjusted gross income rather than the traced $143,835 taxable income. It omitted or materially reduced the $54,670 itemized deduction for mortgage interest and property taxes." +us,scenario_078,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used the $2,400 standard deduction instead of the beneficial $54,670 itemized deduction and retained the $1,632 refund in Maryland AGI. This inflated taxable income to $197,737 instead of $143,835." +us,scenario_078,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model imposed an unsupported $3,004 reduction on the $54,670 itemized deduction and then allowed a residual personal exemption. The traced Maryland computation deducts the full $54,670 from $198,505, leaving $143,835 taxable." +us,scenario_078,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric value for the requested output. It therefore failed the required structured-output contract without performing a usable Maryland tax calculation. +us,scenario_078,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model set the liability to zero despite $143,835 of Maryland taxable income. Applying Maryland’s positive graduated rates to that base produces $6,936.34 before refundable credits." +us,scenario_078,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model explicitly treated the household as having no Maryland-specific listed deductions and used only a standard deduction, ignoring $28,210 of mortgage interest and $26,460 of property taxes. It also submitted $10,933 even though its own initial bracket arithmetic totaled $9,927.78." +us,scenario_078,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly began from federal taxable income and then subtracted a Maryland standard deduction instead of starting with Maryland adjusted gross income and taking the $54,670 itemized deduction. It also invented an unspecified $736 nonrefundable credit and used a rate schedule that does not match Maryland’s stated brackets." us,scenario_078,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_079,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the head's SSI receipt and therefore missed Arizona's automatic Medicaid eligibility pathway for SSI recipients. It treated disability and Indian Health Service coverage as insufficient standalone facts, but the operative PolicyEngine category is SSI_RECIPIENT, which makes head_medicaid_eligible equal to 1." -us,scenario_079,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted current SSDI receipt into Medicare eligibility by assuming completion of the disability waiting period. Constant disability and income during 2026 do not establish the required prior entitlement duration, while the head is only 58 and fails the age test." -us,scenario_079,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model invoked the 24-month SSDI pathway without any fact showing that the head completed 24 months of disability entitlement. The full-year constancy instruction does not supply entitlement history, and age 58 does not satisfy the age-65 pathway." -us,scenario_079,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model treated disabled status plus SSDI income as sufficient for Medicare and skipped the required waiting-period fact. The head is 58, and no completed disability-entitlement period was provided." -us,scenario_079,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model treated receipt of Social Security disability benefits as automatically qualifying the head for Medicare. SSDI receipt alone does not establish completion of the disability waiting period, and the head is under 65." -us,scenario_079,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model cited the 24-month SSDI rule but inferred that its duration requirement was met from one year of constant facts. No prior entitlement duration was listed, and the 58-year-old head fails the age threshold." -us,scenario_079,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"The model explicitly assumed that the standard 24-month waiting period had been met even though that fact was unlisted. With no completed waiting period and age 58, the head is not Medicare eligible." -us,scenario_079,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,The model made SSDI receipt itself sufficient for Medicare eligibility. It omitted the disability-entitlement waiting-period requirement and the head does not qualify by age at 58. -us,scenario_079,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model automatically mapped SSDI income to Medicare eligibility. It did not establish completion of the required disability-entitlement period, and the head is younger than 65." -us,scenario_079,head_medicare_eligible,gemini-3.6-flash,llm_error,age_disability,False,"The model treated disability and SSDI receipt as a complete Medicare eligibility pathway. Those facts do not establish the required entitlement duration, while age 58 fails the benchmark’s age test." -us,scenario_079,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,The model asserted that PolicyEngine makes an under-65 SSDI recipient Medicare eligible. The benchmark computation does not turn the disability and SSDI inputs into eligibility; the head remains below the age-65 threshold. -us,scenario_079,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model treated disabled status alone as sufficient for Medicare before age 65. It omitted the disability-entitlement waiting-period condition, and the 58-year-old head does not meet the age test." -us,scenario_079,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,The model inferred Medicare eligibility from full-year disability and SSDI income. Full-year constancy describes 2026 only and does not establish completion of the prior disability-entitlement period; age 58 is below the qualifying threshold. -us,scenario_079,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,The model treated SSDI receipt as automatically conferring Medicare eligibility. It skipped the required disability-entitlement duration and the head is under age 65. -us,scenario_079,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,"The model invoked eligibility after the 24-month waiting period without any input showing that the period had elapsed. The unlisted duration cannot be assumed, and the head is only 58." -us,scenario_079,head_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model asserted that PolicyEngine makes any disabled SSDI recipient Medicare eligible. It omitted the required entitlement-duration condition, and the head fails the age-65 test." -us,scenario_079,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model correctly named the 24-month disability-entitlement rule but treated current SSDI receipt as proof that it was satisfied. No 24-month history was provided, and the head does not qualify by age." -us,scenario_079,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model invented the fact that the head had received disability income for at least 24 months. The prompt supplies only an annual 2026 amount and constant within-year status, so the waiting period is not established and age 58 does not qualify." -us,scenario_079,head_medicare_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model conflated constant SSDI receipt throughout 2026 with completion of 24 months of disability entitlement. One full tax year does not establish that waiting period, and the head is below age 65." +us,scenario_079,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model ignored that the head receives SSI and therefore belongs to Arizona's automatic SSI-recipient Medicaid category. It incorrectly demanded an additional eligibility pathway or coverage input even though SSI receipt itself establishes Medicaid eligibility. +us,scenario_079,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model assumed that the head had completed the 24-month SSDI waiting period even though no disability-entitlement duration was listed. Constant disability and SSDI receipt during 2026 do not supply that missing prior-period history, so the 58-year-old head remains ineligible." +us,scenario_079,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model converted disability and SSDI receipt directly into Medicare eligibility by assuming completion of the 24-month waiting period. The prompt supplies no such duration, and the head is only 58 rather than at least 65." +us,scenario_079,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model treated being a disabled SSDI recipient as sufficient for Medicare without establishing the required 24 months of disability entitlement. With that unlisted condition false and age 58, the eligibility result is No." +us,scenario_079,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model treated Social Security disability benefits as automatically qualifying the head for Medicare. SSDI receipt alone does not establish completion of the disability waiting period, while the head also fails the age-65 pathway." +us,scenario_079,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model cited the 24-month SSDI rule but never established that the head had received disability benefits for 24 months. Full-year constancy covers 2026 only and does not create the missing entitlement history, so age 58 does not qualify." +us,scenario_079,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"The model explicitly assumed that the standard 24-month waiting period had been met. The prompt requires unlisted statuses to be false, so that assumption is invalid and the 58-year-old head does not meet the age pathway." +us,scenario_079,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,The model equated SSDI receipt with immediate Medicare eligibility. It omitted the disability-entitlement waiting-period requirement and ignored that the head is below age 65. +us,scenario_079,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model treated SSDI receipt as independently sufficient for Medicare. It failed to establish the required disability-entitlement duration, leaving the age-58 head ineligible." +us,scenario_079,head_medicare_eligible,gemini-3.6-flash,llm_error,age_disability,False,"The model treated the disability flag plus SSDI income as automatic Medicare eligibility. Those facts do not establish completion of the 24-month waiting period, and the head is under 65." +us,scenario_079,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model incorrectly stated that disability and SSDI qualify an under-65 person under PolicyEngine rules. This output does not infer the required disability-entitlement history from SSDI income, so the head fails the applicable age threshold." +us,scenario_079,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model treated the disability indicator alone as satisfying a Medicare disability pathway. Disability by itself does not establish Medicare entitlement or completion of its waiting period, and the head is 58." +us,scenario_079,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model inferred Medicare eligibility from full-year disability and SSDI income. A full-year income total does not prove 24 months of disability entitlement, so the under-65 head does not qualify." +us,scenario_079,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model treated receipt of Social Security disability income as sufficient proof of Medicare eligibility. It skipped the disability-entitlement waiting-period condition, while the head also fails the age-65 test." +us,scenario_079,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,"The model invoked eligibility after the 24-month waiting period without evidence that this period had elapsed. Because unlisted entitlement history cannot be inferred, the 58-year-old head is not eligible." +us,scenario_079,head_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,The model asserted that an under-65 disabled SSDI recipient qualifies under PolicyEngine rules. It omitted the required disability-entitlement duration and therefore overrode the age-65 threshold without a qualifying fact. +us,scenario_079,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model correctly identified a 24-month disability-entitlement rule but assumed the head had satisfied it merely because SSDI was listed. The prompt provides no pre-2026 receipt duration, so the under-65 head remains ineligible." +us,scenario_079,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model expressly invented at least 24 months of SSDI receipt. That unlisted history must be treated as absent, and disability plus age 58 does not produce Medicare eligibility." +us,scenario_079,head_medicare_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model misread constant facts throughout the tax year as proof that the head had completed Medicare’s 24-month disability waiting period. Constancy establishes only the 2026 status, not prior entitlement history, so the age-58 head is not eligible." us,scenario_079,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_079,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_079,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model counted only $630 monthly SSDI and omitted the household's $881 monthly SSI from SNAP gross income. It also replaced the traced $354 deduction total with an estimated standard deduction plus $26 medical deduction, substantially understating net income and overstating the allotment." -us,scenario_079,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model treated vehicle equity and speculative home equity as countable SNAP resources. PolicyEngine counts $1,045.29 in assets, passes the asset test, and also establishes categorical eligibility through SSI receipt and TANF non-cash eligibility." -us,scenario_079,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI and calculated gross income from SSDI alone. Its unexplained adjustment from a computed benefit near $5,046 to $4,218 also does not follow the SNAP maximum-allotment-minus-contribution formula." -us,scenario_079,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI from gross income and used an unsupported net-income estimate near $300. The traced calculation instead starts with $1,511 gross income and reaches $1,157 after $354 of deductions." -us,scenario_079,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model counted only $630 monthly SSDI, omitted $881 of SSI, and reduced income using an inaccurately described medical deduction. It also stated that roughly $400 of net income produces a $283 benefit, which does not follow from a $536 maximum allotment and a 30% contribution." -us,scenario_079,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI and therefore used $630 rather than $1,511 as SNAP gross income. It also used only the standard and excess-medical deductions instead of the traced $354 total deductions, causing a large benefit overstatement." -us,scenario_079,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $881 monthly SSI and invented shelter costs from the mortgage balance despite the instruction that unlisted expenses are zero. A loan balance is not a monthly shelter expense, and the traced net income is $1,157 rather than near zero." -us,scenario_079,snap,deepseek-v4-pro,llm_error,asset_resource,False,"The model applied a vehicle-equity resource calculation that PolicyEngine does not apply to this categorically eligible household. The traced countable assets are $1,045.29, so the household passes the asset test through the applicable SSI and TANF non-cash categorical-eligibility treatment." -us,scenario_079,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly included SSDI and SSI but allowed only $224 of monthly deductions, producing $1,211 of net income. PolicyEngine applies $354 of deductions, yielding $1,157, and also uses month-specific 2026 allotment parameters rather than a constant $535 maximum." -us,scenario_079,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated the absence of housing expenses as disqualifying or as forcing a zero benefit. Shelter expenses are not required for SNAP eligibility, and this household qualifies through its income, assets, SSI receipt, and TANF non-cash categorical eligibility." -us,scenario_079,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used $17,220 of annual income instead of the traced $18,132 corresponding to $1,511 per month, then incorrectly concluded that only the minimum allotment applied. With $354 of monthly deductions, net income is $1,157 and the normal allotment exceeds the minimum floor." -us,scenario_079,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used $18,072 of gross income and only $2,712 of annual deductions. The trace uses $1,511 monthly gross income and $354 monthly deductions, and the model also overstated the annual maximum allotment by using $6,720." -us,scenario_079,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used $18,096 of annual gross income rather than the traced $1,511 monthly amount and did not apply the traced $354 monthly deduction total. Those errors produce an incorrect net income and expected contribution." -us,scenario_079,snap,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable SNAP output or explanation. -us,scenario_079,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model incorrectly treated the absence of dependents and shelter costs as preventing a positive SNAP benefit. The two disabled adults satisfy the applicable income and asset tests and qualify categorically through SSI receipt and TANF non-cash eligibility. -us,scenario_079,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the supplied disability, SSDI, SSI-derived eligibility, asset, and household-size facts and asserted that SNAP-enabling information was absent. PolicyEngine has sufficient inputs to establish categorical eligibility and calculate a positive allotment." -us,scenario_079,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model correctly formed $1,511 of monthly gross income but applied only $230 of deductions, yielding $1,281. PolicyEngine applies $354 of deductions and obtains $1,157 of net income, lowering the expected contribution and increasing the benefit." -us,scenario_079,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model awarded the full maximum allotment despite $1,157 of SNAP net income. SNAP subtracts an approximately 30% expected contribution from the maximum, so eligibility does not imply receipt of the full $546 monthly maximum." -us,scenario_079,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's $163 monthly estimate reflects an overstated expected contribution. The traced $1,157 net income follows from $1,511 gross income minus $354 of deductions and produces month-specific allotments totaling $2,428.02 annually." -us,scenario_079,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model counted Social Security and SSI but used the wrong deduction total or contribution calculation to reach $165 per month. The correct net income is $1,157 after $354 of deductions, with month-specific allotments totaling $2,428.02." -us,scenario_079,snap,grok-4.3,llm_error,thresholds_rates,False,"The model asserted that $630 monthly SSDI exceeded the SNAP income limits, even though the traced $1,511 gross income is only 86% of the applicable poverty guideline and $1,157 net income is 66%. The household passes both income tests and receives a positive benefit." -us,scenario_079,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,The model omitted $881 monthly SSI and computed SNAP from SSDI alone. It also used an inflated annual maximum allotment and deductions that do not match the traced $354 monthly total. -us,scenario_079,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,The model invented an SSI amount from an estimated couple rate and exclusion instead of using the $881 SSI amount reflected in the eligibility computation. It then applied only $230 of deductions rather than $354 and used outdated constant allotment parameters. -us,scenario_079,snap,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable SNAP output or explanation. -us,scenario_079,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI and based the benefit solely on $630 of SSDI. It consequently calculated $395 of net income rather than the traced $1,157 and greatly understated the expected contribution." -us,scenario_079,snap,minimax-m3,llm_error,categorical_eligibility,False,The model treated SNAP take-up or prior enrollment as an additional required fact despite the prompt's instruction to assume program take-up when required. The household's eligibility criteria and categorical eligibility produce a calculated benefit without a separately listed enrollment indicator. -us,scenario_079,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI, treated the $35 medical threshold as annual instead of monthly, and used an overstated $598 monthly maximum allotment. These errors reduced net income and inflated the maximum, producing a severe benefit overstatement." -us,scenario_079,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the spouse's SSI-recipient Medicaid category and treated the absence of a separate listed Medicaid input beyond disability and IHS coverage as disqualifying. Under Arizona's SSI-recipient pathway, the spouse's SSI receipt automatically qualifies them for Medicaid, so the correct eligibility value is 1." -us,scenario_079,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,The model treated the spouse's disabled status as sufficient for Medicare eligibility. It missed that Medicare disability eligibility is not triggered by a bare disability flag; this spouse is under 65 and has no listed SSDI-linked Medicare entitlement or qualifying condition such as ESRD or ALS. -us,scenario_079,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model invented an SSDI-equivalent eligibility pathway for the spouse from the disabled and blind facts. The prompt lists SSDI income only for the head, not the spouse, and the spouse is age 55 with no Medicare-entitling disability pathway, so treating disability alone as Medicare eligibility is the exact error." -us,scenario_079,ssi,claude-fable-5,llm_error,thresholds_rates,False,"The model used an estimated 2026 couple rate of $1,450.60 instead of the $1,491 applicable amount. Subtracting $610 from $1,491 yields $881 monthly and $10,572 annually." -us,scenario_079,ssi,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model incorrectly treated the head’s $7,560 annual SSDI as exceeding an individual SSI limit even though it is only $630 monthly, and it failed to apply the eligible-couple computation and spousal deeming. The couple receives $1,491 minus $610 of countable monthly income." -us,scenario_079,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"The model replaced its initial couple-rate estimate with an unsupported $1,940 monthly rate. The applicable couple amount is $1,491, so the benefit is $881 monthly rather than $1,330." -us,scenario_079,ssi,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used an incorrect $1,538 monthly couple rate and then submitted $7,212 despite its own calculation producing $11,136. Using $1,491 less $610 gives $10,572 annually." -us,scenario_079,ssi,claude-opus-5,llm_error,other,False,"The model’s submitted $6,072 does not follow from its own stated subtraction of $7,320 from $17,868, and its stated maximum also differs from the $17,892 annual couple amount. The correct subtraction is $17,892 minus $7,320, yielding $10,572." -us,scenario_079,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model projected a $1,450 monthly couple rate instead of using the applicable $1,491 rate. Its income and resource treatment otherwise leads directly to $1,491 minus $610, or $10,572 annually." -us,scenario_079,ssi,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used an erroneous $1,927 monthly couple rate and then submitted zero despite explicitly concluding that the resource test passed and a positive payment was due. The applicable $1,491 rate less $610 produces $10,572 annually." -us,scenario_079,ssi,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $1,485 monthly couple rate rather than $1,491. The six-dollar monthly understatement produced a $72 annual shortfall." -us,scenario_079,ssi,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the $1,415 couple rate instead of the 2026 amount of $1,491. Subtracting the correctly identified $610 countable income from $1,491 yields $881 monthly." -us,scenario_079,ssi,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly asserted that $610 of monthly countable income exceeds the couple payment standard. It is below the $1,491 eligible-couple amount, leaving an $881 monthly benefit." -us,scenario_079,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $16,980 annual maximum corresponding to a $1,415 monthly rate. The 2026 annual couple amount is $17,892, and subtracting $7,320 yields $10,572." -us,scenario_079,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model understated the annual eligible-couple amount as $17,832 rather than $17,892. This $60 error carried directly into its SSI result." -us,scenario_079,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"The answer implies an annual couple maximum of $17,856 after adding back the correctly computed $7,320 of countable income. The applicable maximum is $17,892, making the benefit $36 higher." +us,scenario_079,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the household's $881 monthly SSI from SNAP gross income and counted only the $630 SSDI. It also replaced the traced $354 monthly deductions with an improvised $235 or $125 deduction, so its net income and allotment were far too favorable." +us,scenario_079,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model counted home and vehicle equity as disqualifying resources even though the traced SNAP resource calculation counts only $1,045.29 and the household also has categorical eligibility through SSI and TANF non-cash eligibility. The mortgage balance supplies no home value from which equity could be inferred, and the prompt forbids inferring unlisted amounts." +us,scenario_079,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI and treated $630 SSDI as the household's entire gross income. The correct gross income is $1,511 and the correct deductions are $354, yielding $1,157 net income rather than its $385 estimate." +us,scenario_079,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted the $881 SSI included in SNAP gross income and then asserted mutually inconsistent net-income and benefit figures. The traced calculation uses $1,511 gross income less $354 of deductions, not $630 less an estimated standard deduction." +us,scenario_079,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model counted only $630 monthly SSDI and omitted $881 monthly SSI from gross income. It also claimed that roughly $400 of net income produced only $283 against a $536 maximum, which does not follow the 30-percent contribution formula." +us,scenario_079,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model's detailed calculation omitted $881 monthly SSI, reducing gross income from $1,511 to $630. It consequently calculated $400 net income instead of the traced $1,157 and overstated the monthly allotment." +us,scenario_079,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI and invented large mortgage shelter costs from the mortgage balance despite the instruction that unlisted expenses are zero. A loan balance is not a deductible monthly shelter payment, and the traced deductions total $354 rather than reducing net income to zero." +us,scenario_079,snap,deepseek-v4-pro,llm_error,asset_resource,False,"The model applied an obsolete vehicle-equity resource shortcut and counted $5,050 of the vehicle as a SNAP asset. The traced resource test counts $1,045.29, below the limit, and the household also meets categorical eligibility through SSI and TANF non-cash eligibility." +us,scenario_079,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly included $630 SSDI and $881 SSI but allowed only $224 of monthly deductions. PolicyEngine applies $354, reducing net income to $1,157 rather than $1,287 or the stated $1,211, so the expected contribution was overstated." +us,scenario_079,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated the absence of housing expenses as eliminating SNAP eligibility or the entire benefit. Shelter expenses affect deductions, but they are not a prerequisite; this household passes the income and asset tests and has categorical eligibility through SSI and TANF non-cash eligibility." +us,scenario_079,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used $17,220 of annual income instead of the traced $18,132 from $630 monthly SSDI plus $881 monthly SSI and failed to apply the full $354 monthly deductions. The normal allotment remains well above the minimum floor, so reducing the result to $276 annually misapplies the benefit formula." +us,scenario_079,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used $18,072 of gross income and only $2,712 of annual deductions. The trace uses $18,132 of gross income and $4,248 of deductions, yielding $13,884 annualized net income before applying month-specific allotment parameters." +us,scenario_079,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used $18,096 of gross income rather than $18,132 and did not apply the traced $354 monthly deduction total. Its $2,136 answer reflects an overstated expected contribution relative to the $1,157 monthly net income." +us,scenario_079,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so it failed the required output contract." +us,scenario_079,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated the absence of dependents and shelter costs as preventing a positive SNAP benefit. Two disabled adults with $1,157 monthly net income qualify, pass the resource test, and receive a positive allotment under the ordinary maximum-minus-contribution formula." +us,scenario_079,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed the supplied facts were inadequate even though the prompt sets every unlisted input to zero and assumes take-up. Disability, SSI receipt, stated income, deductions, and assets establish eligibility and produce $2,428.02." +us,scenario_079,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model correctly formed $1,511 of gross income but deducted only $230 per month, producing $1,281 net income. The traced deductions are $354 and net income is $1,157, so its expected contribution was $37.20 too high per month before accounting for month-specific parameters." +us,scenario_079,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model awarded the full maximum allotment and therefore failed to subtract 30 percent of the household's $1,157 monthly net income. Disability and medical deductions do not reduce countable income to zero." +us,scenario_079,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's $163 monthly estimate reflects deductions below the traced $354 total and therefore net income above $1,157. Applying the full deductions lowers the expected contribution and produces monthly allotments around $199 to $211 as the maximum changes during the year." +us,scenario_079,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model understated the allowable SNAP deductions, producing a $165 monthly allotment instead of using $1,157 net income after $354 of deductions. It also flattened the calculation into a single monthly estimate instead of applying the within-year maximum-allotment change." +us,scenario_079,snap,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly concluded that $630 monthly SSDI could exceed the two-person SNAP income limits. Even after including $881 SSI, gross income is 86 percent of the relevant poverty guideline and net income is 66 percent, so the household passes both tests." +us,scenario_079,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI and calculated gross income from SSDI alone. The correct SNAP gross income is $18,132 annually, not $7,560, and the traced annualized deductions are $4,248." +us,scenario_079,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model invented an $840 monthly SSI amount using an external couple-rate calculation instead of using the traced $881 SSI receipt. It then allowed only $230 in monthly deductions rather than $354, overstating net income and understating SNAP." +us,scenario_079,snap,inkling,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI and based the benefit solely on $630 SSDI. It therefore calculated about $395 of net income rather than the traced $1,157." +us,scenario_079,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so it failed the required output contract." +us,scenario_079,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI and treated $630 SSDI as all gross income. The correct calculation starts from $1,511 gross income and subtracts $354, producing $1,157 net income rather than $395." +us,scenario_079,snap,minimax-m3,llm_error,categorical_eligibility,False,The model ignored the instruction to assume program take-up when required and treated unstated enrollment as a reason to return zero. Eligibility and the positive calculated allotment determine the requested SNAP amount; prior enrollment is not required. +us,scenario_079,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,The model omitted the traced $881 monthly SSI and also applied the $35 medical threshold annually instead of monthly. It used an inflated maximum allotment and annualized the formula without the correct within-year parameters. +us,scenario_079,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly treated SSI receipt as replacing or barring SNAP. SSI instead supports categorical eligibility here, and the household receives the SNAP allotment calculated from $1,157 monthly net income." +us,scenario_079,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated disability and IHS coverage as the only relevant facts and failed to account for the spouse's engine-computed $5,286 of SSI receipt. That receipt places the spouse in Arizona's SSI-recipient category, which automatically confers Medicaid eligibility without further income verification." +us,scenario_079,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted the spouse's disability flag directly into Medicare eligibility and incorrectly inferred an SSDI-linked entitlement. Disability alone is not a Medicare pathway, and the spouse has no listed SSDI receipt or other qualifying basis before age 65." +us,scenario_079,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,The model identified the 24-month SSDI-entitlement requirement but then ignored it and treated disability as sufficient. It also invented SSDI-equivalent status for the spouse even though only the head receives Social Security disability income and unlisted benefit receipt is zero. +us,scenario_079,ssi,claude-fable-5,llm_error,thresholds_rates,False,"The model substituted a projected $1,450.60 monthly couple rate for the applicable 2026 SSI couple standard of $1,491. Using $1,491 minus $610 of countable monthly income yields $881 per month and $10,572 annually." +us,scenario_079,ssi,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model incorrectly tested the head's annual SSDI against an individual SSI limit and declared the entire couple ineligible. Both spouses qualify individually through blindness or disability, and spousal deeming produces $610 of countable monthly income against the $1,491 eligible-couple standard, leaving $881 per month." +us,scenario_079,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"The model abandoned its initial couple-rate calculation and substituted an unsupported $1,940 monthly SSI couple rate. The applicable 2026 rate is $1,491, so subtracting $610 of countable monthly income produces $10,572 annually." +us,scenario_079,ssi,claude-opus-4.8,llm_error,other,False,"The submitted $7,212 contradicts the model's own stated calculation of $11,136 and therefore does not follow from its reasoning. Its stated calculation also used an incorrect $1,538 monthly couple standard; the applicable $1,491 standard less $610 yields $881 monthly and $10,572 annually." +us,scenario_079,ssi,claude-opus-5,llm_error,other,False,"The model stated an annual maximum of $17,868 and countable income of $7,320, whose difference is $10,548, but submitted $6,072. It also used $1,489 rather than the applicable $1,491 monthly couple standard, under which the annual benefit is $10,572." +us,scenario_079,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model estimated the 2026 couple standard as $1,450 per month instead of using $1,491. With its correctly identified $610 of countable monthly income, the payment is $881 per month rather than $840." +us,scenario_079,ssi,claude-sonnet-5,llm_error,other,False,"The model concluded that the resource test passes and explicitly calculated a positive SSI payment, but then submitted zero. Its calculation also used an unsupported $1,927 couple rate; the applicable $1,491 rate less $610 produces $10,572 annually." +us,scenario_079,ssi,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $1,485 monthly couple rate rather than the applicable $1,491. Subtracting $610 from $1,491 gives $881 monthly, six dollars more than its monthly result, or $10,572 annually." +us,scenario_079,ssi,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the $1,415 prior-year couple rate instead of the 2026 standard of $1,491. Applying the same $610 countable-income figure to the correct standard yields $881 per month and $10,572 annually." +us,scenario_079,ssi,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly asserted that countable SSDI exceeds the couple benefit rate. Countable income is $610 per month, below the $1,491 eligible-couple standard, leaving an $881 monthly SSI payment." +us,scenario_079,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a maximum annual couple benefit of $16,980, equivalent to the $1,415 prior-year monthly rate. The 2026 maximum is $17,892, and subtracting $7,320 of annual countable income yields $10,572." +us,scenario_079,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a projected annual couple maximum of $17,832, or $1,486 per month, instead of $17,892, or $1,491 per month. Subtracting $7,320 of countable income from the correct maximum yields $10,572." +us,scenario_079,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted amount implies an annual couple maximum of $17,856, or $1,488 per month, after adding back the stated $7,320 of countable income. The applicable maximum is $17,892, producing $10,572 after the income reduction." us,scenario_079,ssi,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, so the required output was missing." -us,scenario_079,ssi,gpt-5.4-mini,llm_error,asset_resource,False,"The model counted the vehicle toward SSI resources even though the household’s vehicle is excluded. Countable resources are $1,045.29, below the couple limit, so the resource test does not eliminate the $10,572 benefit." -us,scenario_079,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated unlisted current SSI receipt as necessary even though the task requests a calculated benefit and assumes take-up. Both spouses’ listed blindness and disability establish the categorical pathway, after which the income and resource tests yield $10,572." -us,scenario_079,ssi,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a projected $1,487 monthly couple maximum instead of $1,491. The four-dollar monthly understatement reduced the annual result by $48." -us,scenario_079,ssi,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used a $1,486 monthly couple standard rather than $1,491. With $610 countable income, the payment is $881 rather than $876 monthly." -us,scenario_079,ssi,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly concluded that the SSDI exceeds the couple income limit. After the exclusion, countable income is $610 monthly against a $1,491 couple amount, leaving $881 per month." -us,scenario_079,ssi,grok-4.5,llm_error,thresholds_rates,False,"The model used a $1,486 monthly couple rate instead of $1,491. The correct $881 monthly payment annualizes to $10,572." -us,scenario_079,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $1,450 monthly rate, mislabeled it as the 2025 rate, and carried that estimate into 2026. The applicable 2026 couple amount is $1,491, producing $881 monthly after countable income." +us,scenario_079,ssi,gpt-5.4-mini,llm_error,asset_resource,False,"The model counted the vehicle value toward SSI resources even though the household's one vehicle is excluded. Countable resources are $1,045.29, below the couple limit, so both spouses pass the resource test." +us,scenario_079,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated unlisted prior SSI receipt as necessary instead of computing eligibility and assumed take-up as instructed. Both spouses satisfy the aged, blind, or disabled criterion, pass the income and resource tests, and receive $5,286 each." +us,scenario_079,ssi,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a projected $1,487 monthly couple maximum instead of the applicable $1,491. With $610 of countable monthly SSDI, the correct payment is $881 per month rather than $877." +us,scenario_079,ssi,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used a $1,486 monthly couple standard instead of $1,491. Subtracting the correctly identified $610 of countable income from the applicable standard yields $881 per month and $10,572 annually." +us,scenario_079,ssi,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly treated the SSDI income as exceeding the SSI couple limit. After exclusions, countable income is $610 per month against a $1,491 couple standard, leaving $881 per month payable." +us,scenario_079,ssi,grok-4.5,llm_error,thresholds_rates,False,"The model used an estimated $1,486 monthly couple rate rather than the applicable $1,491. The correct rate less $610 of countable monthly income produces $881 monthly and $10,572 annually." +us,scenario_079,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $1,450 monthly couple rate rather than the applicable 2026 standard of $1,491. With $610 of countable monthly income, the payment is $881 per month, not $840." us,scenario_079,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, so the required output was missing." -us,scenario_079,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the absence of listed SSI receipt as a reason to return zero, disregarding the instruction to calculate benefits with assumed take-up. The spouses’ blindness and disability, qualifying resources, and countable income produce $10,572." -us,scenario_079,ssi,qwen-3.7-max,llm_error,thresholds_rates,False,"The model ultimately applied the monthly $20 exclusion correctly but used an estimated $1,450 monthly couple rate. Replacing that estimate with the applicable $1,491 rate gives $10,572 annually." -us,scenario_079,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model incorrectly imposed an earned-income or taxable-filing requirement on Arizona's increased excise tax credit. The two eligible adults generate $25 each despite receiving only nontaxable SSDI, for a total of $50." -us,scenario_079,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that Arizona has no refundable individual income tax credit and treated zero taxable income as disqualifying. Arizona's refundable increased excise tax credit provides $25 per adult to this qualifying two-adult tax unit. -us,scenario_079,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly identified the increased excise tax credit as $25 per person and correctly computed $50 for two adults, but then submitted $200 without any additional qualifying credit. Its stated derivation supports exactly $50, not $200." -us,scenario_079,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly required taxable income or satisfaction of an income-tax filing threshold for the increased excise tax credit. This low-income married couple receives $25 per adult even with no Arizona taxable income, totaling $50." -us,scenario_079,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,The model treated earnings and qualifying dependents as necessary for every Arizona refundable credit. The increased excise tax credit requires neither here and pays $25 for each of the two adults. -us,scenario_079,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly characterized the increased excise tax credit as nonrefundable or inapplicable because the couple has no wages or taxable income. It is refundable and yields $25 for each eligible adult, totaling $50." -us,scenario_079,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,The model considered the property tax credit but omitted the separate refundable increased excise tax credit. The latter provides this two-adult Arizona tax unit with $50 regardless of its ineligibility for the property tax credit. -us,scenario_079,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model focused on dependent-child and property-tax pathways and omitted Arizona's increased excise tax credit. Two qualifying adults receive $25 each, so neither children nor reported property-tax expenses are required for the $50 credit." -us,scenario_079,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The answer implies that the household's income and demographics trigger no refundable program, omitting Arizona's increased excise tax credit. Married filing jointly with two adults produces $25 per adult, totaling $50." -us,scenario_079,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model omitted the eligibility pathway for Arizona's refundable increased excise tax credit. This qualifying two-adult married tax unit receives $25 per adult, totaling $50." -us,scenario_079,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable Arizona credit applies. The increased excise tax credit applies to the two eligible adults at $25 each, producing $50." -us,scenario_079,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model failed to apply Arizona's refundable increased excise tax credit. The household's two qualifying adults generate a $50 credit at $25 each. -us,scenario_079,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model used zero taxable income to exclude refundable Arizona credits and overlooked the increased excise tax credit. That credit does not require positive taxable income here and pays $25 for each adult, totaling $50." -us,scenario_079,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to recognize that the listed Arizona residence, married filing status, low income, and two-adult tax-unit size trigger the increased excise tax credit. The resulting refundable amount is $25 per adult, or $50." -us,scenario_079,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model incorrectly treated earned income or children as necessary supporting facts for any state refundable credit. Arizona's increased excise tax credit requires neither for this household and pays $50 for its two adults. -us,scenario_079,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly required tax liability, earned income, or qualifying children for Arizona's refundable increased excise tax credit. This eligible two-adult couple receives $25 per adult despite lacking all three, totaling $50." -us,scenario_079,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model overlooked that the reported Arizona residence, married filing status, low income, and two-adult tax unit generate the increased excise tax credit. Those facts produce a refundable $50 credit." -us,scenario_079,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model incorrectly made earnings, children, or another special circumstance prerequisites for every Arizona refundable credit. The increased excise tax credit pays this qualifying couple $25 per adult, totaling $50." -us,scenario_079,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model treated the absence of state tax liability as eliminating refundable credits. Arizona's increased excise tax credit is payable independently of positive liability and yields $50 for the two adults. -us,scenario_079,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model incorrectly required qualifying children or positive tax liability for an Arizona refundable credit. The increased excise tax credit applies to this childless two-adult household and pays $25 per adult. -us,scenario_079,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model considered only a state EITC tied to the federal EITC and omitted Arizona's independent increased excise tax credit. That separate credit pays $25 for each of the two eligible adults, totaling $50." -us,scenario_079,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,The model restricted Arizona refundable credits to property-tax or earned-income pathways and therefore demanded unreported expenses or earnings. The increased excise tax credit requires neither and provides $50 for the two adults. -us,scenario_079,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,"The model incorrectly treated childlessness and zero tax liability as disqualifying for all Arizona refundable credits. The increased excise tax credit remains available and pays $25 for each adult, totaling $50." -us,scenario_079,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model incorrectly asserted that Arizona has no state income tax. Arizona levies an individual income tax and provides a refundable increased excise tax credit that gives this two-adult household $50. -us,scenario_079,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model incorrectly used zero Arizona taxable income and no dependents to exclude all refundable credits. Arizona's increased excise tax credit applies to this qualifying two-adult married household and yields $50. +us,scenario_079,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model required SSI receipt to be explicitly listed even though the task directs the model to calculate the benefit and assume program take-up. Both spouses are blind and disabled, pass the financial tests, and therefore receive a combined $10,572." +us,scenario_079,ssi,qwen-3.7-max,llm_error,thresholds_rates,False,"The model ultimately applied the $240 annual general exclusion correctly but used an estimated $1,450 monthly couple maximum. The applicable maximum is $1,491 per month, so $610 of monthly countable income leaves $881 per month." +us,scenario_079,ssi,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model reduced the combined SSI maximum by only $3,780, effectively assigning only half of the head's SSDI to the couple-level reduction. Spousal deeming splits the income for individual computation but does not halve the household's total reduction: combined annual countable income is $7,320, yielding $10,572." +us,scenario_079,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model incorrectly required income or earnings to claim Arizona's increased excise tax credit. The two eligible adults receive $25 each despite having only nontaxable SSDI, yielding $50." +us,scenario_079,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Arizona has no refundable individual credit and treated zero taxable income as disqualifying. Arizona's refundable increased excise tax credit supplies $25 per eligible adult, totaling $50." +us,scenario_079,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly identified the increased excise tax credit and calculated $25 for each of two adults, or $50, but then submitted $200 without any additional qualifying credit. Its final amount contradicts its own derivation." +us,scenario_079,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,The model incorrectly imposed an income-tax filing threshold or taxable-income requirement on the increased excise tax credit. This low-income married couple qualifies for $25 per adult even though Arizona excludes the SSDI from taxable income. +us,scenario_079,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated earnings or dependents as necessary for every Arizona refundable credit. The increased excise tax credit requires neither and pays $25 for each spouse, totaling $50." +us,scenario_079,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model incorrectly classified the increased excise tax credit as nonrefundable and tied refundable-credit eligibility to wages or taxable income. The increased excise tax credit is refundable and provides this couple $50. +us,scenario_079,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model focused on property-tax-credit age and SSI conditions while omitting the separate increased excise tax credit. That credit pays $25 for each of the two eligible adults, totaling $50." +us,scenario_079,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,The model treated dependent children or property-tax-credit eligibility as necessary. Arizona's increased excise tax credit applies to this childless married couple and yields $50. +us,scenario_079,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model overlooked the household's eligibility for Arizona's increased excise tax credit. The two eligible adults generate $25 each, so the refundable-credit total is $50." +us,scenario_079,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model failed to apply Arizona's increased excise tax credit to the low-income married tax unit. The credit contributes $25 per adult, producing $50." +us,scenario_079,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model omitted Arizona's increased excise tax credit. Both spouses qualify for $25, making total state refundable credits $50." +us,scenario_079,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no Arizona refundable credit applied. The increased excise tax credit pays $25 for each of the two adults, totaling $50." +us,scenario_079,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,The model treated zero taxable income as eliminating all broadly available refundable credits. Arizona's increased excise tax credit remains available and supplies $50 to this two-adult tax unit. +us,scenario_079,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model failed to trigger Arizona's increased excise tax credit for the married couple. Its $25-per-adult amount produces $50. +us,scenario_079,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model incorrectly required earned income or children for an Arizona refundable credit. The increased excise tax credit requires neither here and pays $50 for the two adults. +us,scenario_079,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly required tax liability, earned income, or qualifying children. Arizona's refundable increased excise tax credit instead provides $25 for each eligible spouse, totaling $50." +us,scenario_079,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model overlooked that Arizona residence, married filing status, low income, and two adults trigger the increased excise tax credit. The resulting credit is $50." +us,scenario_079,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model wrongly made earnings, children, or another special circumstance prerequisites. The increased excise tax credit applies to this low-income married couple and pays $50." +us,scenario_079,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model incorrectly tied refundable-credit eligibility to positive state tax liability. Arizona's increased excise tax credit is refundable despite zero liability and yields $50. +us,scenario_079,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model incorrectly required qualifying children or positive tax liability. The increased excise tax credit pays $25 to each eligible adult in this childless couple, totaling $50." +us,scenario_079,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model considered only an EITC-style pathway and omitted Arizona's increased excise tax credit. That separate refundable credit does not depend on federal EITC and provides this couple $50. +us,scenario_079,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model treated zero state tax liability as barring refundable credits and omitted the increased excise tax credit. The credit pays $25 per spouse, totaling $50." +us,scenario_079,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model limited Arizona refundable credits to property-tax or earned-income pathways requiring additional inputs. It omitted the increased excise tax credit, which provides $25 for each adult without reported rent, property tax, or earnings." +us,scenario_079,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,The model incorrectly treated childlessness and zero tax liability as disqualifying. Arizona's refundable increased excise tax credit applies to both spouses and totals $50. +us,scenario_079,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model falsely stated that Arizona has no state income tax. Arizona has an individual income tax system and its increased excise tax credit provides this married couple $50. +us,scenario_079,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model incorrectly required Arizona taxable income or qualifying dependents. The increased excise tax credit applies to the low-income two-adult tax unit and yields $50. +us,scenario_079,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model omitted Arizona's increased excise tax credit. At $25 for each eligible spouse, the household receives $50." us,scenario_080,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_080,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model routed the 41-year-old through a disabled-adult or SSI-related Medicaid pathway and ignored Pennsylvania's ACA adult expansion MAGI category. It also treated bank assets and disability-category thresholds as controlling, even though the reference category is ADULT MAGI with income at 0.06 times FPL and no asset test." -us,scenario_080,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model did not provide a parseable head_medicaid_eligible value or explanation. The required output was 1 because the head qualifies under Pennsylvania's ACA adult expansion MAGI category at 0.06 times FPL. -us,scenario_080,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model falsely required additional unspecified immigration or household facts instead of applying the listed and trace-established Medicaid pathway. Under the ADULT MAGI category, the 41-year-old non-dependent head with income at 0.06 times FPL and 5 years in the United States is eligible." -us,scenario_080,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring unspecified extra financial or categorical flags and failed to use the ACA adult expansion MAGI pathway. The small listed investment and assistance income produces MAGI at 0.06 times FPL, which satisfies the Pennsylvania adult expansion income test for this under-65 non-dependent adult." -us,scenario_080,head_medicaid_eligible,grok-4.3,llm_error,asset_resource,False,"The model applied the $2,000 resource limit from a disabled Medicaid pathway to the wrong eligibility category. Pennsylvania's ACA adult expansion category is MAGI-based for this 41-year-old adult, so the $19,828 bank account balance does not bar eligibility when MAGI is 0.06 times FPL." -us,scenario_080,head_medicare_eligible,claude-fable-5,llm_error,health_coverage,False,"The model converted the generic `is disabled` fact into Medicare entitlement and asserted that PolicyEngine treats all disabled people as Medicare eligible. The correct computation does not infer SSDI receipt, 24 months of SSDI entitlement, ESRD, or ALS from the disability flag, so a 41-year-old head remains not Medicare eligible." -us,scenario_080,head_medicare_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model applied a nonexistent rule that disability alone qualifies a person for Medicare regardless of age. For an under-65 adult, disability must connect to a qualifying Medicare pathway such as SSDI entitlement after the waiting period, ESRD, or ALS, none of which is listed." -us,scenario_080,head_medicare_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model treated the disabled status as sufficient for Medicare eligibility at age 41. The correct derivation requires an age-65, ESRD, ALS, or SSDI-entitlement pathway, and the prompt provides only a generic disability flag." -us,scenario_080,head_medicare_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model correctly named the SSDI 24-month pathway but then inferred SSDI receipt and completion of the waiting period from the constant disability status. The prompt instructs not to infer unlisted facts, so no SSDI entitlement history, ESRD, or ALS exists and the 41-year-old head is not Medicare eligible." -us,scenario_080,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model did not provide a parseable `head_medicare_eligible` value or explanation. This is a missing-output contract failure rather than a substantive eligibility calculation. -us,scenario_080,head_medicare_eligible,gpt-5.5,llm_error,health_coverage,False,"The model asserted that PolicyEngine treats disability as qualifying for Medicare despite the head being under 65. PolicyEngine does not turn the generic disability flag into Medicare entitlement without a listed qualifying pathway such as SSDI entitlement after the waiting period, ESRD, or ALS." +us,scenario_080,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model routed the head exclusively through a disabled-adult Medicaid category and required SSI or another disability-linked status. It failed to apply the separate ACA adult expansion pathway, under which the 41-year-old head qualifies at 0.06 FPL regardless of the listed assets or zero SSI receipt." +us,scenario_080,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable output for head_medicaid_eligible. The required eligibility value is 1 because the head meets the ACA adult expansion age, immigration-duration, and MAGI conditions." +us,scenario_080,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated unlisted immigration details and an asserted lack of income information as defeating eligibility. Under the prompt's defaults and the engine inputs, the head satisfies Pennsylvania's five-year immigration condition and has MAGI of $940, or 0.06 FPL, qualifying through the ACA adult expansion category." +us,scenario_080,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required extra financial or categorical flags instead of testing the ACA adult expansion pathway. The head's $940 MAGI equals 0.06 FPL, and the age and immigration-duration requirements are satisfied, so no disability-linked eligibility flag is needed." +us,scenario_080,head_medicaid_eligible,grok-4.3,llm_error,asset_resource,False,"The model applied the $2,000 resource limit from an asset-tested disabled Medicaid pathway. Pennsylvania's MAGI-based ACA adult expansion category does not apply that asset limit, and the 41-year-old head qualifies at 0.06 FPL." +us,scenario_080,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,The model treated the generic disability flag as sufficient for under-65 Medicare eligibility. It improperly inferred SSDI-based entitlement even though SSDI receipt and the required entitlement period were not listed and therefore are false. +us,scenario_080,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model incorrectly asserted that disability qualifies a person for Medicare regardless of age. At age 41, the head needs a specified under-65 Medicare pathway, and none is present." +us,scenario_080,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model equated being disabled with Medicare qualification. The disability indicator does not itself establish SSDI-based Medicare entitlement, ESRD, or ALS, so the 41-year-old head is not eligible." +us,scenario_080,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly named the SSDI waiting-period rule but then assumed that the disability flag proved both SSDI receipt and completion of that period. Constancy of disability during the year does not supply either fact, and unlisted SSDI entitlement, ESRD, and ALS statuses are false." +us,scenario_080,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, violating the required submission contract." +us,scenario_080,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model incorrectly claimed that PolicyEngine converts the generic disability flag directly into Medicare eligibility. Because the head is under 65 and has no listed SSDI-based entitlement, ESRD, or ALS pathway, the eligibility result is 0." us,scenario_080,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_080,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reached zero net income through the utility and excess-shelter deductions but used a flat $293 monthly maximum. The applicable 2026 monthly maximum varies from $298 to $304.68 across the year, producing $3,596.04 annually." -us,scenario_080,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary $2,750 resource limit and incorrectly added financial assistance to resources. Pennsylvania TANF non-cash categorical eligibility governs this household, and its $19,827.65 of countable assets, including vehicle value, is below the applicable Pennsylvania limits." -us,scenario_080,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly treated Pennsylvania categorical eligibility as removing the disqualifying ordinary asset test and derived zero net income, but it used a flat $293 monthly maximum. The 2026 monthly maxima of $298-$304.68 annualize to $3,596.04." -us,scenario_080,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model stopped at the ordinary elderly-or-disabled resource limit. It failed to apply Pennsylvania TANF non-cash categorical eligibility, under which the household passes the applicable asset and income conditions." -us,scenario_080,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model stated that countable income was effectively zero but then selected an unsupported reduced annual allotment of $2,622. Zero net income yields the full 2026 one-person maxima, annualized to $3,596.04." -us,scenario_080,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,The model correctly applied Pennsylvania categorical eligibility and reduced net income to zero with the utility-based excess-shelter deduction. It then used the FY2025-style flat maximum of $292 per month instead of the applicable 2026 monthly values of $298-$304.68. -us,scenario_080,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model applied the ordinary $4,250 disabled-household resource limit despite recognizing very low net income. Pennsylvania TANF non-cash categorical eligibility supplies the qualifying pathway and prevents the stated bank assets from producing a zero benefit." -us,scenario_080,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model explicitly allowed no shelter deduction and therefore charged 30% of $174.33 against the maximum allotment. The $2,000 energy subsidy activates the utility allowance, creating an excess-shelter deduction that reduces net SNAP income to zero." -us,scenario_080,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly reached zero net income but used an approximate $291 monthly maximum. The applicable 2026 one-person monthly allotments are $298-$304.68 and total $3,596.04 for the year." -us,scenario_080,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer treats the assets as disqualifying and omits the Pennsylvania TANF non-cash categorical-eligibility pathway. The household passes that pathway, and the utility-based shelter deduction leaves it entitled to the full annual allotment." -us,scenario_080,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly identified the energy-subsidy utility allowance and zero net income. It used a flat $292 monthly maximum rather than annualizing the applicable 2026 monthly maxima of $298-$304.68. -us,scenario_080,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly applied Pennsylvania categorical eligibility and the utility-based excess-shelter deduction. Its error was fixing the maximum at $292 per month instead of using the 2026 monthly values that total $3,596.04." -us,scenario_080,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly derived categorical eligibility and zero net income but used the wrong maximum-benefit schedule. The 2026 monthly maxima vary from $298 to $304.68 and annualize to $3,596.04, not $3,504." -us,scenario_080,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so it failed the required output contract." -us,scenario_080,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The prompt identifies a one-person household, supplies the relevant income and asset facts, and sets unlisted inputs to zero. The model ignored Pennsylvania categorical eligibility and the stated energy subsidy instead of computing the resulting full allotment." -us,scenario_080,snap,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The model incorrectly claimed that household size was missing even though the head is the sole listed person and all listed people form one household group. SNAP does not require earned income, and the supplied facts establish a qualifying one-person unit." -us,scenario_080,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model excluded the shelter deduction and consequently reduced the maximum allotment by 30% of $96. The energy subsidy activates the utility allowance and excess-shelter deduction, reducing net income to zero and eliminating that benefit reduction." -us,scenario_080,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model explicitly assumed no shelter deduction despite the $2,000 energy subsidy. That subsidy activates the utility allowance, whose excess-shelter deduction reduces net SNAP income to zero and yields the full allotment." -us,scenario_080,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model stopped after subtracting the standard deduction and reduced the allotment based on remaining income. It omitted the energy-subsidy utility allowance and resulting excess-shelter deduction, which reduce net income to zero." -us,scenario_080,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model applied the ordinary disabled-household resource limit directly to the bank balance. Pennsylvania TANF non-cash categorical eligibility is the controlling pathway and allows the household to pass the applicable asset conditions. -us,scenario_080,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed there were no utility facts even though the household has a $2,000 energy subsidy. That subsidy activates the utility allowance and excess-shelter deduction; absence of other shelter expenses does not make SNAP zero." -us,scenario_080,snap,grok-4.5,llm_error,thresholds_rates,False,"The model correctly applied Pennsylvania categorical eligibility and derived zero net income through the utility allowance. It used a flat $292 monthly maximum rather than the applicable 2026 monthly maxima totaling $3,596.04." -us,scenario_080,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model applied the ordinary $4,250 resource limit and ended the eligibility analysis there. Pennsylvania TANF non-cash categorical eligibility allows this household to qualify under the state's applicable asset rules." -us,scenario_080,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated the federal disabled-household resource limit as dispositive. It omitted Pennsylvania TANF non-cash categorical eligibility, through which the household passes the applicable resource conditions." -us,scenario_080,snap,kimi-k3,llm_error,thresholds_rates,False,"The model correctly applied Pennsylvania categorical eligibility and reached zero net income, but it used a flat $292 monthly maximum. The applicable 2026 monthly maximum schedule annualizes to $3,596.04." -us,scenario_080,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model treated assets and the absence of earned income as grounds for zero eligibility without applying Pennsylvania TANF non-cash categorical eligibility. Earned income is not required, and the household qualifies under the state's categorical pathway." -us,scenario_080,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,The model applied the ordinary federal resource limit and incorrectly treated the non-elderly disabled status as excluding the enhanced pathway. Pennsylvania TANF non-cash categorical eligibility governs the household and prevents the stated bank assets from disqualifying it. -us,scenario_080,ssi,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_080,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly determined that the claimant receives 100% Pennsylvania Tax Forgiveness and even computed the resulting liability as $0, but then submitted the $28.86 tentative tax instead. Tax Forgiveness is a nonrefundable credit applied in the requested before-refundable-credits output, so it reduces the liability to $0." -us,scenario_080,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model stopped after applying Pennsylvania’s 3.07% rate to $940 and omitted the Pennsylvania Tax Forgiveness credit. The claimant’s low eligibility income produces 100% forgiveness, reducing the $28.86 tentative tax to $0." -us,scenario_080,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,The model treated the 3.07% tax on $940 as final liability and never applied Pennsylvania Tax Forgiveness. The 100% nonrefundable forgiveness credit eliminates the tentative tax in the before-refundable-credits result. -us,scenario_080,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model calculated and rounded the tentative 3.07% tax but omitted Pennsylvania Tax Forgiveness. At this claimant’s low eligibility income, 100% forgiveness reduces the tentative liability to $0 before refundable credits." -us,scenario_080,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no output for the requested variable, violating the required submission contract." -us,scenario_080,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model submitted the raw 3.07% tax on the $940 of dividends and capital gains without applying Pennsylvania Tax Forgiveness. The claimant qualifies for 100% forgiveness, which reduces that tentative tax to $0 in this output." -us,scenario_080,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"After correctly excluding the $3,600 financial assistance from taxable income, the model stopped at the tentative 3.07% tax on $940 and asserted that no nonrefundable credit applied. Pennsylvania Tax Forgiveness does apply at this income and offsets the tentative tax in full." +us,scenario_080,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reached zero net income but used a flat $293 monthly maximum. PolicyEngine applies the 2026 maximum allotments of $298–$304.68 over their applicable months, yielding $3,596.04 rather than $3,516." +us,scenario_080,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary $2,750 resource limit and even added financial assistance to resources. Pennsylvania's TANF non-cash categorical-eligibility pathway covers this household, whose $19,827.65 of assets is below the applicable Pennsylvania limits, so assets do not eliminate SNAP eligibility." +us,scenario_080,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly removed the asset barrier and derived zero net income, but substituted a flat approximate $293 maximum allotment. The applicable 2026 monthly maxima vary from $298 to $304.68 and annualize to $3,596.04." +us,scenario_080,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model imposed the ordinary elderly-or-disabled household resource ceiling. It omitted Pennsylvania's TANF non-cash categorical-eligibility pathway, under which the household's assets remain below the applicable state limits." +us,scenario_080,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model gave an unsupported range and then used an annual amount equivalent to only $218.50 per month despite describing essentially zero countable income. Zero benefit-reduction income produces the full 2026 allotments of $298–$304.68 per month, totaling $3,596.04." +us,scenario_080,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly used Pennsylvania categorical eligibility and the utility-based shelter deduction to reach zero net income. It then held the maximum allotment at the FY2025-style $292 monthly figure instead of applying the 2026 monthly values of $298–$304.68, which annualize to $3,596.04." +us,scenario_080,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model stopped at the ordinary $4,250 disabled-household resource test. Pennsylvania's TANF non-cash categorical-eligibility rules supersede that shortcut here, and the household's assets are below the applicable Pennsylvania limits." +us,scenario_080,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model explicitly allowed no shelter deduction even though the $2,000 energy subsidy activates the applicable standard utility allowance. That allowance creates an excess-shelter deduction that reduces net SNAP income to zero, producing the full allotment rather than a benefit reduced by 30% of $174.33." +us,scenario_080,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly reached zero net income but used an approximate $291 monthly maximum. PolicyEngine applies the 2026 monthly maxima of $298–$304.68 over their applicable periods, totaling $3,596.04." +us,scenario_080,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The answer treats the household's assets as preventing a positive benefit without applying Pennsylvania's TANF non-cash categorical eligibility. The household passes that pathway and the income tests, so its assets do not reduce SNAP to zero." +us,scenario_080,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly identified the utility allowance, zero net income, and entitlement to the maximum allotment. It used a flat $292 monthly maximum instead of the applicable 2026 values of $298–$304.68, which total $3,596.04." +us,scenario_080,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The eligibility and zero-net-income pathway is correct, but the model used an outdated flat $292 maximum allotment. The applicable 2026 monthly maximum changes during the year between $298 and $304.68 and annualizes to $3,596.04." +us,scenario_080,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly applied categorical eligibility and the heating utility allowance but priced the full benefit at $292 per month. PolicyEngine uses the applicable 2026 maximum allotments of $298–$304.68, producing $3,596.04." +us,scenario_080,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SNAP output or explanation, so it failed the required output contract." +us,scenario_080,snap,gpt-5.4-mini,llm_error,other,False,"The prompt explicitly supplies a one-person household, annual income, assets, disability status, and relevant energy assistance while directing all unlisted inputs to zero. The model incorrectly treated these complete benchmark inputs as missing and defaulted SNAP to zero instead of evaluating eligibility and the allotment." +us,scenario_080,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"SNAP does not require earned income, and the household size is explicitly one. The model ignored the stated composition and Pennsylvania's categorical-eligibility pathway, then defaulted to zero rather than calculating the full allotment." +us,scenario_080,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model explicitly assumed no shelter deduction despite the $2,000 energy subsidy, which activates the applicable utility allowance and excess-shelter deduction. That deduction reduces net income to zero, so no 30% income reduction applies; the remaining calculation uses the 2026 monthly maxima and totals $3,596.04." +us,scenario_080,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model explicitly omitted a shelter deduction even though the energy subsidy activates the standard utility allowance. Applying the resulting excess-shelter deduction reduces net income to zero and entitles the household to the full 2026 allotment of $3,596.04." +us,scenario_080,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model stopped after subtracting only the standard deduction and therefore reduced the allotment for residual income. It omitted the utility allowance triggered by the energy subsidy and the resulting excess-shelter deduction, which reduces net income to zero." +us,scenario_080,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model applied the ordinary disabled-household resource limit directly to the bank balance. Pennsylvania's TANF non-cash categorical-eligibility pathway applies, and the household's assets are below the applicable Pennsylvania limits." +us,scenario_080,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model said there were no utility expenses even though the stated $2,000 energy subsidy activates the applicable standard utility allowance. It also treated missing shelter expenses as grounds for a zero benefit, whereas low countable income and the utility-based shelter deduction produce the full allotment." +us,scenario_080,snap,grok-4.5,llm_error,thresholds_rates,False,"The model correctly applied Pennsylvania categorical eligibility and the utility-based excess-shelter deduction. It then used a flat $292 monthly maximum rather than the applicable 2026 monthly amounts of $298–$304.68, which total $3,596.04." +us,scenario_080,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model imposed the ordinary $4,250 resource ceiling and stopped. Pennsylvania's TANF non-cash categorical-eligibility pathway applies to this low-income household, whose assets are below the applicable state limits." +us,scenario_080,snap,inkling,llm_error,categorical_eligibility,False,The model incorrectly applied an ordinary resource ceiling rather than Pennsylvania's TANF non-cash categorical-eligibility pathway. School-age children are irrelevant to SNAP eligibility for this one-person disabled household. +us,scenario_080,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model applied the federal disabled-household resource limit as dispositive. Pennsylvania's TANF non-cash categorical eligibility instead covers this household, and its assets fall below the applicable Pennsylvania limits." +us,scenario_080,snap,kimi-k3,llm_error,thresholds_rates,False,"The model correctly reached categorical eligibility and zero net income but used a flat $292 maximum allotment. The applicable 2026 monthly maximum is $298–$304.68 across the year, totaling $3,596.04." +us,scenario_080,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model treated assets and the absence of earnings as disqualifying without applying Pennsylvania's TANF non-cash categorical eligibility. Earned income is not required, and the household passes the applicable income, asset, and categorical-eligibility rules." +us,scenario_080,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly imposed the ordinary $2,750 resource ceiling on a household qualifying through Pennsylvania's TANF non-cash categorical-eligibility pathway. Under the applicable Pennsylvania rules, the stated assets do not bar eligibility." +us,scenario_080,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the household's resources as exceeding a conventional SNAP limit and failed to apply Pennsylvania's TANF non-cash categorical eligibility. The absence of earned income is not disqualifying, and this household qualifies for the full allotment." +us,scenario_080,ssi,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SSI output or explanation, violating the required structured-output contract." +us,scenario_080,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the prompt's general disability fact as establishing SSI disability, despite the applicable engine input is_ssi_aged_blind_disabled=False. Because the 41-year-old is neither SSI-aged, blind, nor SSI-disabled, the federal base-rate and unearned-income calculation does not apply and SSI is $0." +us,scenario_080,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly calculated 100% Pennsylvania Tax Forgiveness and explicitly derived a $0 post-credit liability, then contradicted that derivation by submitting the uncredited $28.86 gross tax. Tax Forgiveness is nonrefundable and therefore reduces state income tax before refundable credits to $0." +us,scenario_080,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,The model stopped after applying Pennsylvania’s 3.07% rate to $940 and omitted the household’s 100% Tax Forgiveness credit. That nonrefundable credit fully offsets the $28.86 gross liability in the requested before-refundable-credits output. +us,scenario_080,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,The model computed only Pennsylvania’s gross tax on the capital gains and dividends and failed to apply 100% Tax Forgiveness. The forgiveness credit is nonrefundable and reduces the requested liability from the gross tax to $0. +us,scenario_080,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model asserted that no deductions applied and returned the rounded gross tax, but it omitted Pennsylvania’s 100% Tax Forgiveness credit. That nonrefundable credit eliminates the liability before refundable credits; rounding $28.86 to $29 is also inconsistent with the requested amount calculation." +us,scenario_080,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, violating the required submission contract." +us,scenario_080,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model applied the 3.07% rate to $940 and returned the resulting gross tax without applying Pennsylvania Tax Forgiveness. The household receives 100% forgiveness, which fully offsets that gross liability as a nonrefundable credit." +us,scenario_080,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model correctly excluded the $3,600 financial assistance from taxable income but incorrectly stated that no nonrefundable credits applied. Pennsylvania’s 100% Tax Forgiveness credit fully offsets the $28.86 gross tax, leaving $0 before refundable credits." us,scenario_080,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model limited SALT to the listed $5,643 real-estate tax and omitted deductible Massachusetts income tax, understating itemized deductions by $8,484.41. It then used approximate bracket thresholds and reported a number inconsistent with its own $28,759 calculation." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted unreimbursed employee business expenses above the line, used an obsolete $14,600 standard deduction, and failed to itemize $31,028.65 of mortgage-interest and SALT deductions. Its submitted $29,825 also does not follow from its stated $30,956.92 computation." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model imposed a $10,000 SALT cap instead of deducting $14,127.41, leaving taxable income $4,127.41 too high. Its $31,369 submission also contradicts its own bracket calculation of $27,887." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model capped SALT at $10,000 rather than using the $14,127.41 deductible amount, so it overstated taxable income. It then submitted $31,379 despite explicitly calculating $28,172." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model counted only $5,643 of property tax and omitted deductible Massachusetts income tax, reducing itemized deductions from $31,028.65 to $22,543. It also replaced its own roughly $28,968 bracket calculation with an unsupported $31,650 estimate." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated the listed $5,643 real-estate tax as the entire SALT deduction and omitted $8,484.41 of deductible Massachusetts income tax. That shortcut produced $151,551 of taxable income instead of $143,059.47." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Although the model approximated itemized deductions near the correct scale, it did not actually apply the 2026 single-filer brackets to its $147,194 taxable-income figure. The stated tax from that base is nowhere near $33,700, so the submitted value reflects a rate-schedule computation error." -us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,The model wrongly reduced the stated annual wages by the employer-sponsored insurance premium and introduced a personal exemption. It also used a pre-TCJA rate schedule and omitted deductible Massachusetts income tax from SALT. -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model introduced a $5,050 personal exemption that is not part of the reference computation and used pre-TCJA rates. Its $30,823.10 itemized-deduction estimate also differs from the traced $31,028.65 amount." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,000 standard deduction even though $31,028.65 of itemized deductions is optimal. This left taxable income far above the traced $143,059.47." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $7,389 insurance premium from wages and applied a personal exemption. It also used an expired pre-TCJA rate schedule instead of the applicable 2026 brackets." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model double-counted the employer insurance premium as a wage reduction, added an inapplicable personal exemption, and omitted deductible Massachusetts income tax from SALT. It then taxed that erroneous base under an expired-law framework." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced the stated wages by $7,389 and recognized only $22,543 of itemized deductions, omitting $8,484.41 of deductible Massachusetts income tax. Those offsetting errors produced an incorrect taxable-income base and tax." -us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model applied a personal exemption and a restored pre-TCJA rate schedule, neither of which belongs in the traced computation. Its estimated SALT and total itemized deductions also differ from the exact $14,127.41 and $31,028.65 amounts." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The answer implies a substantial underapplication of the 2026 single-filer rate schedule: $143,059.47 of taxable income yields $26,932.27, not $22,450. Its explanation supplies no bracket arithmetic capable of producing the submitted amount." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model conflated the standard and itemized deductions as amounts to subtract together or otherwise overstated the taxable base; only the better deduction method is used. With $31,028.65 itemized and a $1.24 QBI deduction, taxable income is $143,059.47 rather than a base capable of generating $41,495." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,The model introduced an estimated personal exemption even though the traced calculation has none. It also used approximate itemized deductions and submitted a tax inconsistent with the applicable 2026 schedule. -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the stated annual wages by the employer-sponsored insurance premium. The listed wages enter AGI at $175,002; after the traced deductions, the resulting tax is $26,932.27 rather than $24,920." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model identified the correct deduction categories but used estimated rather than exact 2026 parameters. Exact itemized deductions of $31,028.65, a $1.24 QBI deduction, and the applicable brackets yield $26,932.27, leaving its result $60.32 too high." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified itemization and the 24% marginal bracket but did not apply the exact deduction and bracket parameters. The traced taxable income of $143,059.47 produces $26,932.27, not $26,966.21." -us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's roughly $151,500 taxable income reflects itemizing only mortgage interest and listed property tax while omitting deductible Massachusetts income tax. Correct itemized deductions reduce taxable income to $143,059.47." -us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly restored a personal exemption and the pre-TCJA rate schedule. The applicable computation uses no personal exemption, deducts $31,028.65 of itemized expenses plus $1.24 of QBI, and applies the 2026 single-filer brackets." -us,scenario_081,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model omitted deductible Massachusetts income tax, recognizing only $22,543 rather than $31,028.65 of itemized deductions. It compounded that error by applying a pre-TCJA 10/15/25/28% rate schedule." -us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract." -us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model limited itemized deductions to mortgage interest and $5,643 of real-estate tax, omitting $8,484.41 of deductible Massachusetts income tax. This produced taxable income of $151,543.80 instead of $143,059.47." -us,scenario_081,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model returned zero despite positive taxable income and supplied no computation. After $31,028.65 of itemized deductions and the $1.24 QBI deduction, $143,059.47 remains taxable and generates $26,932.27 of federal tax." -us,scenario_081,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model falsely stated that $22,543 of itemized deductions is less than a $15,000 standard deduction and therefore selected the standard deduction. It also omitted deductible Massachusetts income tax, and its final $39,827.88 contradicts its own $31,281.32 bracket total." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model included only the $5,643 real-estate tax in SALT and omitted deductible Massachusetts income tax, reducing itemized deductions from $31,028.65 to $22,543. It also abandoned its own bracket calculation and submitted $29,385 instead of the amount its stated arithmetic produced." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted unreimbursed employee business expenses above the line, used a stale $14,600 standard deduction, and failed to itemize the deductible mortgage interest and Massachusetts SALT totaling $31,028.65. It then submitted $29,825 despite explicitly calculating $30,956.92, creating an additional arithmetic inconsistency." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model imposed a $10,000 SALT cap instead of using the $14,127.41 deductible SALT amount, leaving taxable income $4,127.41 too high. Its submitted $31,369 also contradicts its own stated bracket calculation of about $27,887." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model capped SALT at $10,000 rather than deducting $14,127.41, so it understated itemized deductions by $4,127.41. It then replaced its own $28,172 bracket result with an unsupported $31,379 submission." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model deducted only mortgage interest and real-estate tax, omitting deductible Massachusetts income tax and thereby using $22,543 rather than $31,028.65 of itemized deductions. Its final $31,650 also falls outside the roughly $28,968 result produced by its own displayed rate calculation." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated the listed real-estate tax as the entire SALT deduction and omitted deductible Massachusetts income tax, overstating taxable income by about $8,485. It also included the $6 REIT/PTP amount in AGI even though the traced AGI is $174,088.12 and that income instead supports the $1.24 QBI deduction." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Although the model came close to the deduction structure, it applied the 2026 single-filer brackets incorrectly: $147,193 of taxable income does not generate $33,700 under the stated rate schedule. It also used a rounded $10,000 SALT amount instead of $14,127.41 and omitted the precise $1.24 QBI deduction." +us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $7,389 employer-sponsored premium from the stated annual wages and restored a personal exemption that does not apply in the traced 2026 calculation. It also omitted deductible Massachusetts income tax from SALT and used an obsolete pre-TCJA rate schedule." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted a $5,050 personal exemption from taxable income and used restored 15%, 25%, and 28% brackets. The correct computation uses $31,028.65 of itemized deductions, a $1.24 QBI deduction, and the 2026 10%, 12%, 22%, and 24% brackets applicable here." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,000 standard deduction and never applied the larger $31,028.65 itemized deduction comprising mortgage interest and deductible SALT. That shortcut left taxable income far above $143,059.47." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a TCJA-expiration regime with a personal exemption and restored pre-TCJA brackets. The traced 2026 calculation instead applies $31,028.65 of itemized deductions, the $1.24 QBI deduction, and the current 10%/12%/22%/24% single-filer schedule to $143,059.47." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model double-deducted the $7,389 employer-sponsored insurance premium from stated wages, claimed an inapplicable $5,300 personal exemption, and omitted Massachusetts income tax from SALT. Those errors replaced the traced $174,088.12 AGI and $31,028.65 itemized deduction with an incorrect tax base." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced stated wages by the separately listed $7,389 employer-sponsored insurance premium and omitted deductible Massachusetts income tax from SALT. The correct taxable income is $143,059.47 after $31,028.65 of itemized deductions and a $1.24 QBI deduction." +us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model restored a $5,150 personal exemption and applied pre-TCJA 15%, 25%, and 28% brackets. It also estimated SALT rather than using the traced $14,127.41 amount; the applicable calculation taxes $143,059.47 under the 2026 single-filer schedule." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The unexplained $22,450 is inconsistent with taxing the traced $143,059.47 of taxable income under the 2026 single-filer brackets. Even after itemizing mortgage interest and SALT with no credits, the bracket calculation yields $26,932.27, not the submitted amount." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $41,495 submission is incompatible with the model's stated deductions and the applicable single-filer brackets. Applying the 2026 schedule to $143,059.47 after $31,028.65 of itemized deductions and the $1.24 QBI deduction yields $26,932.27." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model inserted an estimated personal exemption even though none applies and used an imprecise $30,933 itemized deduction instead of $31,028.65. Those tax-base errors, followed by an incorrect rate calculation, produced $29,417.08 rather than taxing $143,059.47 correctly." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the stated wage amount by the separately listed employer-sponsored insurance premium. The engine's AGI begins with the full $175,002 of employment income and reaches $174,088.12 before $31,028.65 of itemized deductions and the $1.24 QBI deduction." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model identified the correct categories of income and itemized deductions but did not apply their precise traced amounts and bracket thresholds. The exact deductions leave $143,059.47 taxable, whose 2026 single-filer tax is $26,932.27 rather than $26,992.59." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model recognized that mortgage interest and Massachusetts/property taxes are itemized but miscomputed the resulting bracket tax. Taxable income of $143,059.47 under the exact 2026 single-filer thresholds produces $26,932.27, not $26,966.21." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's $151,500 taxable-income estimate reflects itemizing mortgage interest and property tax while omitting deductible Massachusetts income tax. Including total SALT of $14,127.41 and the $1.24 QBI deduction lowers taxable income to $143,059.47." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and the pre-TCJA 15%, 25%, and 28% brackets. The traced calculation has no personal exemption and applies the 2026 10%, 12%, 22%, and 24% brackets to $143,059.47." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model omitted deductible Massachusetts income tax from itemized deductions, leaving taxable income near $151,551 rather than $143,059.47. It compounded that error by applying obsolete pre-TCJA 15%, 25%, and 28% brackets." +us,scenario_081,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model substantially captured the itemized deductions and QBI deduction but used estimated amounts and bracket thresholds, producing taxable income around $143,215 instead of $143,059.47. Applying the exact 2026 thresholds to the traced taxable income yields $26,932.27 rather than $27,056." +us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so the required structured answer was missing." +us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model deducted only mortgage interest and real-estate tax, omitting deductible Massachusetts income tax and understating itemized deductions by $8,485.65. This left taxable income at $151,543.80 instead of $143,059.47." +us,scenario_081,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The zero answer ignores the positive tax liability generated by $174,088.12 of AGI. With $31,028.65 of itemized deductions, a $1.24 QBI deduction, and no offsetting credits, the bracket computation yields $26,932.27." +us,scenario_081,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model falsely stated that $22,543 of itemized deductions was lower than a $15,000 standard deduction and therefore selected the standard deduction. It also omitted deductible Massachusetts income tax, and its final $39,827.88 contradicts its own displayed tax total of $31,281.32." +us,scenario_081,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly selected the standard deduction even though its own $22,543 partial itemized total already exceeded it. It also omitted deductible Massachusetts income tax, so it never reached the correct $31,028.65 itemized deduction or $143,059.47 taxable income." us,scenario_081,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_081,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model correctly calculated all three components, including $805.01 of Massachusetts PFML, but then discarded PFML and submitted a number that does not even equal its stated $13,387.65 federal subtotal. The required sum is $10,850.12 + $2,537.53 + $805.01 = $14,192.66." -us,scenario_081,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model omitted the mandatory 0.46% Massachusetts PFML employee contribution of $805.01 and used an incorrect Social Security wage-base assumption despite wages being fully taxable for Social Security. Its submitted $13,383.41 also contradicts its own component arithmetic." -us,scenario_081,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model excluded the $805.01 Massachusetts PFML employee contribution and then submitted $12,321.73 even though its own federal components sum to $13,387.65. Adding PFML produces $14,192.66." -us,scenario_081,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model treated Massachusetts PFML as absent and omitted its $805.01 employee contribution. Its submitted $12,717.62 also fails to equal the correctly stated Social Security and Medicare subtotal of $13,387.65." -us,scenario_081,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model set the Massachusetts PFML employee contribution to approximately zero instead of applying 0.46% to $175,002. It therefore omitted $805.01, and its $13,385 submission also contains an arithmetic or rounding error relative to the $13,387.65 federal subtotal." -us,scenario_081,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model ultimately applied a 0.604% Massachusetts PFML employee rate instead of the 0.46% rate used for this payroll-tax computation. That inflated PFML from $805.01 to $1,057.01 and overstated the total by $252." -us,scenario_081,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model incorrectly characterized Massachusetts PFML as employer-side and omitted the mandatory $805.01 employee contribution. It also submitted $12,318.65 despite explicitly deriving a $13,387.65 federal payroll-tax subtotal." -us,scenario_081,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model used a 0.38% Massachusetts PFML employee rate instead of 0.46%. The correct state contribution is $805.01, not about $665, bringing total payroll tax to $14,192.66." -us,scenario_081,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model applied an estimated 0.318% Massachusetts PFML rate and calculated only $556.51 of state payroll tax. The applicable 0.46% contribution is $805.01, so it understated the total by $248.50." -us,scenario_081,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model omitted the $805.01 Massachusetts PFML contribution and submitted a value that does not equal 7.65% of the stated $175,002 wage base. Social Security and Medicare alone equal $13,387.65 before PFML raises the total to $14,192.66." -us,scenario_081,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced payroll-tax wages from $175,002 to $167,613 by subtracting the listed employer-sponsored insurance premiums, even though the trace assesses all three components on $175,002. It then used an unspecified estimated PFML amount rather than the 0.46% contribution of $805.01." -us,scenario_081,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly reduced Social Security and Medicare wages to $167,613 and applied only an approximate 0.24% Massachusetts PFML rate. The computation uses $175,002 for all three bases and a 0.46% PFML contribution, yielding $14,192.66." -us,scenario_081,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model subtracted $7,389 of employer-sponsored insurance premiums from wages without establishing that they were pre-tax payroll deductions, reducing the FICA base to $167,613. It also omitted the mandatory $805.01 Massachusetts PFML contribution; the trace applies all components to $175,002." -us,scenario_081,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model used a 0.532% Massachusetts PFML employee rate and calculated about $931 instead of applying 0.46%. The correct PFML amount is $805.01, so its total is overstated by roughly $126." -us,scenario_081,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model included only Social Security and Medicare and omitted the mandatory $805.01 Massachusetts PFML employee contribution. Its submitted $13,389.37 also differs from the exact federal subtotal of $13,387.65." -us,scenario_081,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model incorrectly said $175,002 exceeds the $200,000 Additional Medicare Tax threshold and produced $5,420 without a valid component calculation. Wages are below that threshold, while ordinary Social Security, Medicare, and Massachusetts PFML sum to $14,192.66." -us,scenario_081,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model correctly computed the $13,387.65 federal subtotal but estimated Massachusetts PFML at $962.51. Applying 0.46% to $175,002 gives $805.01, reducing the total to $14,192.66." -us,scenario_081,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model used the combined 2.9% Medicare rate instead of the 1.45% employee share, thereby including the employer share in an employee-side output. It also omitted the distinct 0.46% Massachusetts PFML employee contribution of $805.01." -us,scenario_081,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model used a 0.318% Massachusetts PFML employee rate and calculated only about $556. The applicable 0.46% rate produces $805.01, making total payroll tax $14,192.66." -us,scenario_081,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,The model asserted that Massachusetts imposes no applicable state payroll tax and included only the 7.65% federal FICA taxes. It omitted the mandatory $805.01 Massachusetts PFML employee contribution. -us,scenario_081,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so the required output was missing." -us,scenario_081,payroll_tax,kimi-k3,llm_error,payroll_tax_base,False,"The model applied a 0.256% Massachusetts PFML rate, producing $448.01 instead of $805.01. The applicable 0.46% rate raises the total from its $13,835.66 to $14,192.66." -us,scenario_081,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer omitted every employee payroll-tax component. The wage facts generate $10,850.12 of Social Security tax, $2,537.53 of Medicare tax, and $805.01 of Massachusetts PFML tax." -us,scenario_081,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly excluded Massachusetts PFML and correctly derived only the $13,387.65 federal subtotal, then submitted $14,525.24 under the guise of rounding. The required addition is $805.01 of PFML, producing $14,192.66 rather than the unexplained submitted figure." +us,scenario_081,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model explicitly calculated the correct $805.01 Massachusetts PFML contribution but then excluded it on the false premise that PolicyEngine normally counts only federal payroll taxes. Its submitted $13,268.70 also does not equal its stated federal subtotal of $13,387.65." +us,scenario_081,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an incorrect $168,600 Social Security wage base for 2026 and omitted the mandatory $805.01 Massachusetts PFML contribution. Its submitted $13,383.41 is also inconsistent with its own stated component total of $12,990.73." +us,scenario_081,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model omitted the $805.01 Massachusetts PFML employee contribution after treating it as approximately zero. Its submitted $12,321.73 also contradicts its correctly computed federal payroll-tax subtotal of $13,387.65." +us,scenario_081,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model set the Massachusetts PFML contribution to zero even though it is a mandatory employee state payroll tax included in the requested output. Its submitted $12,717.62 also fails to equal the $13,387.65 federal subtotal shown in its reasoning." +us,scenario_081,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model omitted the 0.46% Massachusetts PFML employee contribution of $805.01. It also rounded its federal components to an unsupported $13,385 instead of the exact $13,387.65 subtotal." +us,scenario_081,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model replaced the applicable 0.46% Massachusetts PFML employee rate with 0.604%, overstating the state contribution by $252.00. The applicable $805.01 contribution produces $14,192.66, which the model itself computed before abandoning it." +us,scenario_081,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model incorrectly treated Massachusetts PFML as employer-side and omitted the employee contribution of $805.01. Its submitted $12,318.65 also contradicts its own federal subtotal of $13,387.65." +us,scenario_081,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a 0.38% Massachusetts PFML rate instead of 0.46%, understating the state contribution. It then rounded its $14,052.66 calculation to a whole dollar rather than returning the annual amount to cents." +us,scenario_081,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model applied a 0.318% Massachusetts PFML rate and calculated only $556.51. The required 0.46% rate produces an $805.01 state contribution. +us,scenario_081,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model omitted the $805.01 Massachusetts PFML contribution and submitted a value that does not equal 6.2% plus 1.45% of $175,002. Those stated federal components alone equal $13,387.65, before adding PFML." +us,scenario_081,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced payroll-taxable wages to $167,613 by subtracting the listed $7,389 employer-sponsored insurance premiums, although the trace taxes the full $175,002. It also failed to apply the exact 0.46% Massachusetts PFML contribution of $805.01." +us,scenario_081,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly reduced Social Security and Medicare wages from $175,002 to $167,613 by subtracting the insurance-premium amount. It also used a 0.24% Massachusetts PFML rate instead of 0.46%." +us,scenario_081,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model subtracted $7,389 of employer-sponsored insurance premiums from wages even though Social Security, Medicare, and Massachusetts PFML are assessed on the full $175,002 in the trace. It also omitted the $805.01 PFML contribution entirely." +us,scenario_081,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"The model used a 0.532% Massachusetts PFML employee rate instead of 0.46%, producing $931 rather than $805.01. This overstated total payroll tax by roughly $126." +us,scenario_081,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model counted only Social Security and Medicare and omitted the mandatory $805.01 Massachusetts PFML employee contribution. Its $13,389.37 figure also does not equal the correct federal subtotal of $13,387.65." +us,scenario_081,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model incorrectly stated that $175,002 exceeds the $200,000 Additional Medicare Tax threshold and produced a total unrelated to the applicable components. Social Security and regular Medicare alone total $13,387.65, and Massachusetts PFML adds $805.01." +us,scenario_081,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly calculated $13,387.65 of federal payroll tax but used an unsupported $962.51 Massachusetts PFML contribution. Applying 0.46% to $175,002 yields $805.01." +us,scenario_081,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"The model applied the combined 2.9% Medicare rate instead of the 1.45% employee rate, thereby including the employer share. It also omitted the separately applicable 0.46% Massachusetts PFML employee contribution." +us,scenario_081,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,The model used a 0.318% Massachusetts PFML rate and calculated only $556. The applicable 0.46% rate yields $805.01. +us,scenario_081,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly asserted that Massachusetts has no applicable state payroll tax and therefore omitted the $805.01 PFML employee contribution. Federal Social Security and Medicare total $13,387.65 before that contribution is added." +us,scenario_081,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so the required output was missing." +us,scenario_081,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,"The model used a 0.256% Massachusetts PFML rate, yielding $448.01 instead of $805.01. The applicable employee rate is 0.46% of $175,002." +us,scenario_081,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer omitted every applicable component: $10,850.12 of Social Security tax, $2,537.53 of Medicare tax, and $805.01 of Massachusetts PFML tax. These components total $14,192.66." +us,scenario_081,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly excluded the mandatory Massachusetts PFML employee contribution of $805.01. Its submitted $14,525.24 also contradicts its own stated $13,387.65 component total and is not a rounding of that amount." +us,scenario_081,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model failed to identify and add the $805.01 Massachusetts PFML employee contribution and gave no component arithmetic supporting $13,922. Social Security and Medicare equal $13,387.65, with no Additional Medicare Tax, and PFML brings the total to $14,192.66." us,scenario_081,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own component calculation produced about $8,238.40, then it replaced that result with $8,380 without identifying any additional taxable component. That unsupported final adjustment caused the error." -us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented $3,835 of dependent, property-tax, and personal-exemption credits for a single taxpayer with no dependents. Massachusetts instead applies the $4,400 personal exemption as a subtraction in deriving Part B taxable income, not as thousands of dollars of nonrefundable credits." -us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model deducted an estimated $11,287 of FICA from Massachusetts income instead of using the deductions embedded in the traced $169,058 Part B income before exemption. It then abandoned its own $7,966 calculation and raised the answer to $8,730 without a corresponding tax component." -us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model taxed wages, interest, and dividends together after only the personal exemption, leaving taxable income at $170,768. It missed the deductions that reduce Part B income before exemption to $169,058 and failed to preserve the separate $110.12 Part A dividend calculation." -us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model stated a taxable base of about $168,768 and tax of about $8,438, then submitted $8,620 based on unspecified adjustments. It never derived the traced $164,658 Part B taxable income or the separate Part A dividend tax." -us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $3,000 renter deduction and no employee payroll-tax deduction, yielding $167,774 rather than the traced $164,658 of Part B taxable income. It also folded dividends into its ordinary-income schedule instead of separately taxing $110.12 as Part A income." -us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model calculated about $8,539 after applying only the personal exemption, then reduced the answer to $8,200 through nonexistent part-year and rounding adjustments. The facts specify a full-year Massachusetts resident, and rounding cannot account for that reduction." -us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model imported a $166,705 federal AGI, deducted $22,543 of itemized deductions, and used an estimated $5,000 exemption. Massachusetts instead derives $169,058 of Part B income before the fixed $4,400 personal exemption and separately taxes Part A dividends." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied assumed $2,000 payroll-tax and $3,000 renter deductions directly to wages, producing $165,602. The traced Part B calculation produces $164,658 after exemption, and the final liability also includes tax on $110.12 of Part A dividends." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer applies 5% directly to roughly $175,000 of gross income. It omitted the deductions that produce $169,058 of Part B income before exemption, the $4,400 personal exemption, and the separate Part A dividend calculation." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model's stated deductions from its stated base do not equal its claimed $157,219 taxable income, so it removed an additional unexplained amount. The correct Part B base after the $4,400 exemption is $164,658, with $110.12 taxed separately under Part A." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model began from a federal taxable-wage figure of $167,613 and treated the capital loss as offsetting interest and dividends. Massachusetts uses $169,058 of Part B income before exemption and retains $110.12 of taxable Part A dividends." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model began from an unsupported $167,613 wage base and then subtracted the personal exemption, renter deduction, and payroll-tax deduction again. This double reduction produced $157,213 instead of $164,658 of Part B taxable income and omitted the separate Part A dividend tax." -us,scenario_081,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model allowed the $1,080 short-term capital loss to offset ordinary income and omitted the $4,400 personal exemption while applying only a $3,000 renter deduction. It also failed to use the Part A/Part B split that leaves $110.12 of dividends taxable separately and $164,658 of Part B taxable income." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The submitted $6,440 implies a 5% taxable base near $128,800, far below the traced $164,658 Part B taxable income. The response supplied no deductions capable of producing that reduction and omitted the separate tax on $110.12 of Part A dividends." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $13,951 answer implies an effective liability near 8% of gross income despite Massachusetts applying 5% to the relevant Part A and Part B components at this income level. It also failed to derive the $164,658 Part B taxable base after exemption." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used $164,602 rather than the traced $164,658 Part B taxable income and incorrectly eliminated all $166 of interest and dividends with the short-term capital loss. The computation retains $110.12 of Part A dividend income, whose tax raises the result above the submitted Part B-only amount." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The $7,915 answer implies a taxable base near $158,300 after applying the 5% rate, so the model deducted substantially more than the traced calculation. The applicable Part B taxable income is $164,658, and $110.12 of Part A dividends is taxed separately." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model reduced income using its assumed maximum renter deduction but did not reproduce the traced $169,058 Part B income before exemption. It consequently used the wrong Part B taxable base and omitted the precise separate tax on $110.12 of Part A dividends." -us,scenario_081,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied 5% to approximately $174,094 without subtracting the $4,400 personal exemption or the deductions embedded in Part B income before exemption. It also failed to separate the $110.12 Part A dividend component from Part B income." -us,scenario_081,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model treated the $1,782 of unreimbursed employee expenses as fully deductible and used a net-income starting point that included the capital loss against ordinary income. Those choices yielded $167,912 rather than $164,658 of Part B taxable income and did not preserve $110.12 of separately taxable Part A dividends." -us,scenario_081,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model subtracted only the $4,400 personal exemption from a $174,094 net-income figure. It missed the deductions producing $169,058 of Part B income before exemption and improperly collapsed the Part A dividend and capital-loss treatment into one base." -us,scenario_081,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_081,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model subtracted only a $2,000 payroll-tax deduction and the $4,400 exemption from a $174,088 aggregate base, producing $167,688. It missed the remaining adjustment needed to reach $169,058 before exemption and failed to tax $110.12 of Part A dividends separately." -us,scenario_081,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model returned zero despite the household being a Massachusetts resident with substantial Massachusetts-taxable wages. Massachusetts taxes $164,658 of Part B taxable income and $110.12 of Part A dividends, so treating the household as having no state liability discarded the applicable state tax entirely." -us,scenario_081,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model netted the $1,080 short-term capital loss against wages and investment income, then subtracted only the personal exemption. Massachusetts instead derives $169,058 of Part B income before exemption and retains $110.12 of dividends as separately taxable Part A income." +us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own component calculation produces approximately $8,238, but it discarded that result and submitted $8,380 through an unexplained $142 addition. The correct Part A and Part B computations total $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It invented $3,835 of dependent, property-tax, and other nonrefundable credits for a single filer with no dependents and supplied no qualifying credit pathway. It also treated unreimbursed employee expenses as an itemized Massachusetts deduction instead of using Part B taxable income of $169,058 before the $4,400 exemption." +us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It deducted an estimated $11,287 of FICA from Massachusetts Part B income and then abandoned the resulting $7,966 computation for an unexplained $8,730 estimate. The trace instead yields $169,058 before exemption, $164,658 after exemption, plus tax on $110.12 of Part A dividends." +us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It taxed wages, interest, and dividends together after only the personal exemption, omitting the adjustments that reduce Part B taxable income before exemption to $169,058. It also failed to preserve the separate Part A tax calculation on $110.12 of dividends." +us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its stated deductions imply about $8,438, yet it submitted $8,620 by adding unspecified adjustments. The required category computation produces Part B taxable income of $164,658 after exemption and a total tax of $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used a $3,000 renter deduction and otherwise built the base directly from listed income, producing $167,774 after exemption. The applicable Part B adjustments instead yield $169,058 before the $4,400 exemption, or $164,658 after it, while $110.12 of dividends is taxed separately under Part A." +us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It correctly calculated about $8,539 from its chosen base, then reduced that figure to $8,200 using nonexistent part-year adjustments and unspecified rounding. The household is a full-year resident, and the traced Part A and Part B liabilities total $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It imported a federal AGI of $166,705 and invented $22,543 of Massachusetts itemized deductions plus a $5,000 exemption. Massachusetts Part B taxable income is $169,058 before the actual $4,400 exemption, and dividend income is handled separately under Part A." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It started solely from wages and subtracted assumed $2,000 payroll-tax and $3,000 renter deductions, yielding the wrong Part B base. The trace gives $169,058 before exemption and $164,658 afterward, with a separate Part A tax on $110.12 of dividends." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It applied 5% directly to roughly $175,000 without calculating Massachusetts taxable income, the $4,400 personal exemption, or the separate Part A component. The correct Part B base after exemption is $164,658, and the combined liability is $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its claimed $157,219 base does not follow from the listed $4,400, $4,000, and $2,000 deductions and substantially understates traced Part B taxable income. Part B taxable income after exemption is $164,658, with additional Part A tax on $110.12 of dividends." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It substituted a supposed federal taxable-wage figure of $167,613 for the Massachusetts Part B calculation and treated the capital loss as offsetting interest and dividends. The Massachusetts trace yields $169,058 before exemption and separately retains $110.12 of taxable Part A dividends." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It began from an unsupported $167,613 wage base and then cumulatively deducted rent, payroll tax, and the personal exemption to reach $157,213. The trace instead produces $169,058 before the $4,400 exemption and $164,658 afterward, plus Part A tax." +us,scenario_081,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It offset the $1,080 short-term capital loss against ordinary income, deducted only $3,000 of rent, and omitted the $4,400 personal exemption. Massachusetts category rules instead produce $169,058 of Part B taxable income before exemption and $110.12 of taxable Part A dividends." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its $6,440 answer implies a taxable base near $128,800, far below the traced $164,658 Part B base after exemption. It therefore applied excessive or unsupported deductions and omitted the correct separate Part A computation." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $13,951 answer is incompatible with the 5% Massachusetts liability on this income and implies either a much larger base or an inapplicable higher rate. Income is below the million-dollar surtax threshold, and the traced Part A and Part B taxes total $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It computed Part B taxable income as $164,602 rather than the traced $164,658 and then eliminated all interest-and-dividend tax through the capital-loss offset. The trace retains $110.12 of Part A dividend income; Part B tax on $164,658 plus that Part A tax equals $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $7,915 estimate implies only $158,300 of income taxed at 5%, materially below the traced $164,658 Part B base after exemption. It over-deducted the income base and failed to add the separate Part A dividend tax." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It reduced income using only a personal exemption and a supposed maximum renter deduction, without reconstructing the traced Part B adjustments or Part A category. The correct bases are $164,658 after the Part B exemption and $110.12 of Part A dividends." +us,scenario_081,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It applied 5% to approximately $174,094 without subtracting the Massachusetts personal exemption or other adjustments embedded in Part B taxable income. The traced Part B base after exemption is $164,658, and dividends are taxed separately under Part A." +us,scenario_081,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It treated the $1,782 of unreimbursed employee expenses as fully deductible and built the tax base from net income after the capital loss. Those steps do not reproduce Massachusetts Part B taxable income of $169,058 before exemption, and the trace separately taxes $110.12 of Part A dividends." +us,scenario_081,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used net federal-style income of $174,094 and subtracted only the $4,400 personal exemption. Massachusetts Part B taxable income is $169,058 before exemption, not $174,094, and the $110.12 dividend component is calculated separately under Part A." +us,scenario_081,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It started from a federal-style $174,094 base and imposed an estimated $3,000 rent deduction, yielding $166,694 after exemption. The trace instead yields $164,658 of Part B taxable income after exemption and separately taxes $110.12 of Part A dividends." +us,scenario_081,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It supplied no numeric output or explanation for the requested variable, so the answer could not be evaluated as a substantive calculation." +us,scenario_081,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted only a $2,000 payroll-tax deduction and the $4,400 exemption from the overall income figure, while asserting there was no rent input despite the listed $44,400 rent. This bypassed the traced Part B calculation of $169,058 before exemption and omitted the separate Part A dividend tax." +us,scenario_081,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It returned zero despite the household being a Massachusetts resident with substantial Massachusetts-taxable wages. The state computation produces $164,658 of Part B taxable income after exemption plus $110.12 of Part A dividends, for $8,238.41 of tax." +us,scenario_081,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It netted the short-term capital loss into a federal-style $174,094 income base and subtracted only the personal exemption. Massachusetts instead produces $169,058 of Part B taxable income before exemption and separately taxes $110.12 of Part A dividends." +us,scenario_081,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its $7,800 estimate implies a taxable base of about $156,000 and provides no deductions that produce that figure. The traced Part B base is $164,658 after exemption, with a separate Part A dividend component bringing total tax to $8,238.41." us,scenario_081,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_082,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly placed the one-year-old within the applicable CHIP age range. The engine's CHIP criteria exclude this child on age, and Medicaid eligibility is also false, so the output is 0." -us,scenario_082,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model omitted the required child1_chip_eligible output. The required result is 0 because the one-year-old fails CHIP's age criterion and is not Medicaid-eligible. -us,scenario_082,child1_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model substituted a broad characterization of New York Child Health Plus coverage for the benchmark's CHIP eligibility test and treated under age 19 as sufficient. The engine excludes this one-year-old under CHIP's age criterion, and no Medicaid eligibility pathway applies." -us,scenario_082,child1_early_head_start_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model explicitly converted the benchmark output into an age-only question by reading ""as a child"" as asking only whether the child meets the under-age-3 criterion. It acknowledged the household income exceeds Early Head Start limits but then discarded that required income test, so it returned eligible from age status alone." -us,scenario_082,child1_early_head_start_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model misread ""as a child"" as limiting the output to the child-level age criterion and excluded the required family income screen from the eligibility determination. The correct benchmark output requires both Early Head Start categorical conditions and income eligibility, and the household's approximately $90,132 income fails the income threshold." -us,scenario_082,child1_early_head_start_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model asserted that the child meets both age and income criteria, which contradicts the required income calculation. The household income used by PolicyEngine is approximately $90,132, above the Early Head Start income threshold, so the income criterion fails even before any age-only shortcut can make the child eligible." -us,scenario_082,child1_early_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated Early Head Start eligibility as satisfied solely because a one-year-old is under the program's age ceiling. It omitted the required household income eligibility step, which fails because approximately $90,132 of household income exceeds the Early Head Start threshold." -us,scenario_082,child1_early_head_start_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model recognized that Early Head Start uses categorical and income criteria but then applied only the under-age-3 criterion to the PolicyEngine output. It failed to carry through the income test, under which this high-income household is not eligible despite the child's age." -us,scenario_082,child1_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model computed a household MAGI above both 160% FPL and 400% FPL, then reversed its own income-threshold conclusion by asserting that New York's generous child rules still make the child Medicaid eligible. It conflated broader New York child coverage or CHIP-style generosity with Medicaid eligibility and ignored that the child qualifies through no Medicaid category at 5.43 times FPL." -us,scenario_082,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model understated Medicaid MAGI by treating the large capital loss as reducing income enough to fall under a roughly 405% FPL child threshold. PolicyEngine's MAGI computation places the household at 5.43 times FPL, so the child exceeds the applicable Medicaid income pathway and has no Medicaid category." -us,scenario_082,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated New York Child Health Plus as Medicaid and concluded eligibility from age and residence without applying the Medicaid category and MAGI screen. At 5.43 times FPL, the 1-year-old qualifies through no Medicaid pathway, so Child Health Plus-style coverage does not make child1 Medicaid eligible." -us,scenario_082,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model asserted that New York child coverage rules cover a 1-year-old at this income level without calculating the household's MAGI-to-FPL ratio. The correct Medicaid screen uses 5.43 times FPL, which is above the applicable Medicaid thresholds for the child, leaving medicaid_category as NONE." -us,scenario_082,child1_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,The model relied on an infants-under-19 expanded-coverage pathway and Child Health Plus-style reasoning instead of the Medicaid eligibility categories used by PolicyEngine. Employer-sponsored insurance is not the deciding step here; the decisive step is that household MAGI is 5.43 times FPL and no Medicaid category applies. +us,scenario_082,child1_chip_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly treated age 1 as within the applicable CHIP age range. The trace applies CHIP's age criterion against the child and returns false, while Medicaid eligibility also returns NONE." +us,scenario_082,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the requested child1_chip_eligible output and supplied no explanation, violating the required submission contract." +us,scenario_082,child1_chip_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model substituted a broad description of New York Child Health Plus for the benchmark's applicable CHIP eligibility test and therefore treated every child under 19 as eligible. The trace rejects this 1-year-old under CHIP's age criterion, and the child has no Medicaid eligibility pathway because Medicaid returns NONE." +us,scenario_082,child1_early_head_start_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model explicitly reinterpreted “as a child” as asking only whether the child met the general age range and ignored the benchmark’s full eligibility determination. It also acknowledged that household income exceeded the program limit but failed to apply that disqualifying income test. +us,scenario_082,child1_early_head_start_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model treated “as a child” as restricting the output to age eligibility, even though the requested variable is overall Early Head Start eligibility for the child. It omitted the failed categorical condition and disregarded the household’s approximately $90,132 income, which exceeds the applicable limit." +us,scenario_082,child1_early_head_start_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly asserted that the household met the income criterion. Program income of approximately $90,132 exceeds the Early Head Start threshold, and the household also lacks a qualifying categorical or pregnancy pathway." +us,scenario_082,child1_early_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stopped after observing that age 1 falls within Early Head Start’s general service age range. It failed to apply the additional categorical and income conditions, both of which evaluate to false for this household." +us,scenario_082,child1_early_head_start_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model recognized that Early Head Start requires categorical and income criteria but then substituted satisfaction of the age ceiling for satisfaction of overall eligibility. The household fails the categorical condition, has no pregnancy pathway, and exceeds the income threshold." +us,scenario_082,child1_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model calculated that household income exceeded even its stated 400% FPL child-coverage threshold, then contradicted that calculation by asserting Medicaid eligibility without identifying any qualifying pathway. The child’s Medicaid category is NONE at 5.43 times FPL." +us,scenario_082,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model treated the large capital loss as reducing Medicaid MAGI enough to fall below a purported 405% FPL limit. The applicable MAGI calculation produces 5.43 times FPL, not an income level below that threshold, and the child qualifies through no Medicaid category." +us,scenario_082,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model treated New York Child Health Plus as Medicaid and declared eligibility without applying the Medicaid category tests. Child Health Plus is a separate child-coverage pathway; the engine assigns this child Medicaid category NONE at 5.43 times FPL. +us,scenario_082,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model assumed that being age 1 in New York establishes Medicaid eligibility at this income without testing the applicable income limit. The household is at 5.43 times FPL, above the available Medicaid thresholds, leaving the child with no qualifying category." +us,scenario_082,child1_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model relied on Child Health Plus-style expanded coverage as though it established Medicaid eligibility and never identified a Medicaid category satisfied by the child. Employer-sponsored insurance does not create a Medicaid pathway, and at 5.43 times FPL the child’s Medicaid category is NONE." us,scenario_082,child1_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_082,child1_wic_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model correctly recognized the child’s categorical WIC age eligibility but incorrectly concluded that the household met the WIC income requirement. It failed to compare the household’s roughly $90,132 annual income against the 185% FPL WIC limit for a two-person household, which the income exceeds." -us,scenario_082,child1_wic_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated the child’s under-5 status as sufficient for WIC eligibility and omitted the separate household income test. It never applied the 185% FPL WIC financial threshold, under which this two-person household’s roughly $90,132 annual income is too high." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's submitted $14,748 contradicts its own successive calculations of $12,383 and $10,353. It also used a $2,000 CTC instead of $2,200 and failed to compute the traced $12,363.05 pre-credit tax followed by $2,800 of nonrefundable credits." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model used the single-filer standard deduction even though the filer qualifies as head of household. It also applied $1,920 of CDCC to all $9,600 of childcare rather than the one-dependent $3,000 expense cap and submitted $23,437 despite deriving only $4,986 after its stated credits." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model estimated a $23,500 standard deduction and approximate head-of-household brackets, producing $12,936 before credits instead of $12,363.05. With the traced deductions and exact 2026 schedules, the same $2,013.60 preferential tax and $2,800 of credits yield $9,563.05." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The submitted $11,150 does not follow from the model's own final calculation of $10,460. Its calculation also used estimated deductions and brackets and a $2,000 CTC instead of the traced $2,200 credit." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model substituted estimated head-of-household bracket cutoffs and a $24,500 standard deduction for the exact 2026 parameters. Those approximations overstated pre-credit tax by $366.95; the traced $12,363.05 less $2,800 equals $9,563.05." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the $13,424 qualified dividends when constructing AGI, reporting $106,519 rather than including dividends in total income before the preferential-rate computation. It also used estimated deductions and only a $2,000 CTC, driving taxable income and final tax below the traced amounts." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model explicitly calculated $10,739 after credits but submitted $13,920. The submitted value therefore abandons its own computation and does not apply the traced $12,363.05 pre-credit tax less $2,800 of credits." -us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used obsolete 10/15/25 percent brackets, personal exemptions, a $1,000 CTC, and unsupported $26,228 itemized deductions. The applicable computation uses the 2026 ordinary brackets, no personal exemptions, the $24,150 standard deduction, and a $2,200 CTC." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model doubled the listed $9,217 real-estate tax into $18,564 of SALT, claimed unsupported personal exemptions, and used only a $1,000 CTC. The filer instead takes the $24,150 standard deduction and receives $2,200 of nonrefundable CTC." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model supplied no derivation supporting its asserted $100,571 AGI and used a $2,000 CTC. The trace produces $92,735.15 of taxable income, $12,363.05 before credits, and a $2,200 CTC plus $600 CDCC." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model invented $14,298 of itemized deductions and $10,400 of personal exemptions, neither of which applies to the traced computation. It also mentioned only the $600 CDCC and failed to apply the $2,200 CTC correctly." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied personal exemptions in addition to the head-of-household standard deduction and used only a $1,000 CTC. The traced calculation has no personal exemption and subtracts $2,200 of CTC plus $600 of CDCC from $12,363.05." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer implies an unsupported personal-exemption reduction and does not identify or apply the full $2,800 of nonrefundable credits. The correct sequence starts from $92,735.15 taxable income and reduces $12,363.05 of pre-credit tax to $9,563.05." -us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $23,063 standard deduction and estimated bracket thresholds rather than the exact $24,150 deduction and 2026 schedules. It also used a $2,000 CTC instead of $2,200, leaving tax $898.33 too high." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model gave only a rough aggregate estimate and did not reproduce the qualified-dividend worksheet or the actual nonrefundable credits. The traced calculation is $10,349.45 of ordinary tax plus $2,013.60 of preferential tax, less $2,800 of credits." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model relied on unspecified itemized deductions even though the $24,150 head-of-household standard deduction governs. Its rounded estimate also fails to account for the traced preferential treatment of $13,424 of qualified dividends and the full $2,800 of nonrefundable credits." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model claimed approximately $24,415 of itemized deductions without support from the listed deductible expenses and then used a $2,000 CTC. The trace applies the $24,150 standard deduction and $2,200 CTC, yielding $9,563.05." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model reduced federal AGI to about $89,951 by treating the full net capital loss as currently deductible. The traced computation instead produces $92,735.15 of taxable income and $12,363.05 before the $2,800 of credits." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Although the model identified the correct credit amounts, its estimate used an imprecise AGI and tax schedule and overstated the result by $170.95. The exact ordinary and qualified-dividend calculations total $12,363.05 before credits." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model named the correct $2,200 CTC and $600 CDCC but implicitly calculated $13,036 before credits rather than $12,363.05. It misapplied the 2026 head-of-household ordinary-rate calculation on the taxable income underlying the qualified-dividend worksheet." -us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model provided no supporting computation and its $14,000 estimate implies that deductions, preferential dividend rates, or the full nonrefundable credits were omitted. The traced calculation yields $12,363.05 before credits and $9,563.05 after them." -us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted employer-sponsored insurance premiums from AGI, added personal exemptions, and declared the CTC fully phased out below the $200,000 phaseout threshold. The trace instead applies $2,200 of CTC and no personal exemption." -us,scenario_082,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used a post-sunset $12,591 standard deduction instead of the applicable $24,150 head-of-household deduction and omitted the $2,200 CTC entirely. Those two errors substantially overstated taxable income and final liability." -us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric value or explanation for the requested output. -us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model subtracted a $700 non-itemizer charitable deduction from taxable income in addition to the standard deduction. That unsupported extra deduction lowered ordinary tax by $153.09, producing $9,579.96 instead of the traced $9,563.05 after the exact computation." -us,scenario_082,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the $30,711 net capital loss as capable of eliminating wages and dividend income. The traced computation retains $92,735.15 of taxable income, so the capital loss, standard deduction, and CTC do not reduce liability to zero." -us,scenario_082,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the entire $30,711 net capital loss in its AGI shortcut, used an estimated $21,900 standard deduction, and treated only $400 of the CTC as nonrefundable. With ample tax liability, the full $2,200 CTC is used nonrefundably, and the submitted $12,866.90 also contradicts every intermediate result in its reasoning." -us,scenario_082,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model applied the ACTC earned-income cap as if it independently created a $1,700 refundable credit. It missed that ACTC is the refundable portion of unused CTC after the nonrefundable CTC is applied against federal income tax, and this household has enough pre-refundable tax liability to use the child credit nonrefundably, leaving refundable CTC at $0." -us,scenario_082,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated one age-1 child as generating a fully refundable $4,000 Child Tax Credit under a rule that does not govern the 2026 reference calculation. It also failed to separate the nonrefundable CTC from federal_refundable_credits; the household's CTC is used against tax liability and produces no refundable ACTC." -us,scenario_082,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no parseable value or explanation for federal_refundable_credits. The required output is $0 because all refundable federal credit components evaluated to zero, including EITC and the refundable portion of CTC." -us,scenario_082,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model made the same ACTC cap-as-payment error by assigning $1,700 of refundable CTC solely because there is one qualifying child and sufficient earned income. It ignored that the nonrefundable CTC is first used against federal income tax, and with remaining tax liability to absorb the credit there is no unused CTC to refund." -us,scenario_082,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model used an outdated or wrong refundable CTC cap of $1,600 and treated that cap as automatically available once earned income exceeded $2,500. The correct computation applies the child credit nonrefundably against the household's federal income tax first, leaving no unused CTC and therefore $0 refundable ACTC." +us,scenario_082,child1_wic_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly asserted that the household met the WIC income requirement. It failed to compare approximately $90,132 of annual household income with the much lower 185%-of-poverty limit for a two-person household." +us,scenario_082,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the child's age under five as sufficient for WIC eligibility. Age establishes categorical eligibility only; the household must also pass the 185%-of-poverty income test, which it fails at approximately $90,132 of annual income." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The submitted $14,748 contradicts the model's own final calculation of roughly $10,353 and therefore reflects an abandoned intermediate result rather than its stated computation. It also omitted the 2026 $2,200 CTC and the deduction step that brings taxable income to $92,735.15." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model used the single-filer standard deduction even though the filer qualifies as head of household. It also treated the $9,600 childcare cost as fully creditable instead of applying the $3,000 one-person CDCC expense cap, and its submitted value contradicts its own arithmetic." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used estimated rather than applicable 2026 head-of-household deduction and bracket values, producing $12,936 before credits instead of $12,363.05. The correct $600 CDCC and $2,200 CTC then yield $9,563.05." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used a $23,000 standard deduction and a $2,000 CTC instead of the applicable $24,150 deduction and $2,200 credit. Its submitted $11,150 also contradicts the $10,460 result stated in its explanation." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model substituted estimated head-of-household brackets and a $24,500 standard deduction for the applicable 2026 parameters. Those estimates overstated pre-credit tax; the traced ordinary and preferential-rate taxes total $12,363.05 before the $2,800 of credits." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the $13,424 qualified dividends when forming AGI, reducing AGI to $106,519 and taxable income to $84,019. Qualified dividends remain included in AGI and taxable income even though they receive preferential tax rates." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The submitted $13,920 does not follow from the model's own stated $13,339 pre-credit tax and $2,600 of credits, which produce $10,739. The underlying estimate also used an understated standard deduction and outdated $2,000 CTC." +us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly itemized $26,228, claimed personal exemptions, and used obsolete 10/15/25 percent brackets. It also allowed only a $1,000 CTC instead of $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model doubled the available SALT amount to $18,564, subtracted nonexistent personal exemptions, and itemized instead of using the $24,150 standard deduction. It also reduced tax by only a $1,000 CTC rather than $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The stated $100,571 AGI omits taxable dividend income or otherwise nets investment items incorrectly; the traced gross-income and deduction steps lead to $92,735.15 of taxable income. It also used a $2,000 CTC rather than $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model reduced income to $96,446, claimed $10,400 of personal exemptions, and used unsupported itemized deductions. The applicable calculation uses the $24,150 standard deduction and no personal exemption." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated AGI at $96,454 and claimed personal exemptions that do not enter the traced calculation. It also applied only a $1,000 CTC instead of the applicable $2,200 credit." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer implies excessive deductions, including the personal exemptions cited in its explanation, because the correct taxable-income derivation produces $12,363.05 before credits. No personal exemption is subtracted in this computation." +us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $23,063 standard deduction and estimated bracket thresholds instead of the applicable $24,150 deduction and 2026 rate schedule. It also used a $2,000 rather than $2,200 CTC." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer does not account correctly for the $3,000 capital-loss limit, $24,150 standard deduction, and preferential-rate treatment that produce $12,363.05 before credits. Subtracting the applicable $600 CDCC and $2,200 CTC yields $9,563.05, not $12,719." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model relied on itemized medical, OTC, real-estate-tax, and charitable expenses even though those deductions do not exceed the applicable standard deduction and the medical expenses do not clear the AGI floor. This left substantially too much tax relative to the traced $92,735.15 taxable income." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The claimed $24,415 of itemized deductions is unsupported by the listed expenses; the calculation instead uses the $24,150 standard deduction and reaches $92,735.15 of taxable income. The model's resulting pre-credit tax was therefore $133.95 too high." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The $89,951 AGI treats the full $30,710.82 net capital loss as currently deductible. Only $3,000 offsets ordinary income in the year, so taxable income and tax are far higher than the model calculated." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Although the model identified the main income items and both credits, it used estimated deduction or bracket values that produced $12,534 before credits. The applicable ordinary and preferential schedules produce $12,363.05 before the same $2,800 reduction." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The answer implies $13,036 of pre-credit tax after the stated $2,800 of credits. Applying the actual 2026 head-of-household brackets and qualified-dividend stacking to $92,735.15 yields $12,363.05 before credits." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The answer does not implement the traced income, deduction, preferential-rate, and credit calculation. The correct sequence yields $12,363.05 before credits and $9,563.05 after the $600 CDCC and $2,200 CTC." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly excluded $21,208 of employer-sponsored insurance premiums from the stated wages, claimed personal exemptions, and treated the CTC as fully phased out. The wages input already reflects the benchmark's gross wage amount, and income is well below the $200,000 CTC phaseout threshold." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used a $12,591 standard deduction instead of the $24,150 head-of-household deduction and omitted the $2,200 CTC entirely. Those errors substantially overstated taxable income and final tax." +us,scenario_082,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model's ordinary-rate computation produced $10,638 instead of the traced $10,349.45. Applying the applicable 2026 head-of-household bracket thresholds to the traced taxable income eliminates the $287.95 overstatement." +us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model subtracted a separate $700 non-itemizer charitable deduction on top of the $24,150 standard deduction and consequently used $92,812 rather than the traced $92,735.15 taxable income. Its ordinary-rate tax calculation also exceeded the traced $10,349.45 by $16.91." +us,scenario_082,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the full $30,711 net capital loss as available to erase current income. The annual deduction against ordinary income is limited to $3,000, leaving $92,735.15 of taxable income and substantial federal tax." +us,scenario_082,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the entire $30,711 net capital loss in AGI instead of limiting the current-year deduction to $3,000. It also treated the refundable cap on the child credit as leaving only $400 nonrefundable, whereas the filer has enough liability to use the full $2,200 CTC nonrefundable." +us,scenario_082,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model overstated AGI to $139,775, used the single-filer standard deduction, and invented a $6,237 QBI deduction despite no qualified business income. It also failed to use head-of-household status and the full $2,200 CTC." +us,scenario_082,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated the $1,700 ACTC ceiling as an automatic refund while simultaneously acknowledging that the filer had sufficient tax liability to use the CTC nonrefunded. Because the CTC is fully absorbed against pre-refund tax, no unused CTC remains for the ACTC, so refundable credits are $0." +us,scenario_082,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model incorrectly treated the child credit as fully refundable and assigned $4,000 for one child. The CTC first offsets income tax as a nonrefundable credit, and this filer's liability absorbs it completely, leaving a refundable portion of $0." +us,scenario_082,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for federal_refundable_credits, violating the required output contract." +us,scenario_082,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated $1,700 as an automatic refundable CTC amount and incorrectly asserted that the capital loss caused a refundable portion. Even after the allowed capital-loss treatment, the filer has enough federal tax liability to use the CTC nonrefunded, leaving no ACTC." +us,scenario_082,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model equated satisfying the ACTC earned-income threshold with receiving the full refundability cap. That threshold only limits a refundable balance left after applying the CTC against tax; here the full credit is used nonrefunded, so the ACTC is $0." +us,scenario_082,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model assigned an approximate $1,875 ACTC solely from the presence of a qualifying child and sufficient earned income. It omitted the requirement that CTC remain unused after offsetting federal income tax; this filer's liability absorbs the credit, producing no refundable ACTC." us,scenario_082,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_082,local_income_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model treated `state: NY` as NYC residency and applied New York City resident income tax brackets to a taxable-income estimate. The household facts do not place the family in NYC, so no NYC local income tax applies and the local-income-tax output remains zero." -us,scenario_082,local_income_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model estimated NYC personal income tax from AGI solely because the household is in New York. The prompt supplies New York State residence but no NYC locality, and the local-income-tax variable only charges NYC tax when that local jurisdiction applies." -us,scenario_082,local_income_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"The model computed NYC head-of-household local tax on an invented NYC taxable income base. It missed that the household's specified location is only NY state, with no NYC residence fact, so the NYC income tax pathway is not triggered." -us,scenario_082,local_income_tax,glm-5.2,llm_error,state_local_rule,False,"The model explicitly assumed the household resides in NYC because NYC is the relevant New York locality with a local income tax. That assumption contradicts the benchmark inputs: `state: NY` does not imply NYC residency, so its NYC tax, school tax credit, and NYC child and dependent care credit calculation is outside the applicable local-income-tax rules." -us,scenario_082,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model correctly calculated $7,664.92 of federal FICA but explicitly excluded New York PFL and disability contributions. Those mandatory state payroll taxes add $411.91 and $31.20." -us,scenario_082,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,The model used an incorrect approximate 0.62% New York PFL charge and then inserted unspecified additional state withholdings. The required New York components are $411.91 of PFL and $31.20 of disability contributions. -us,scenario_082,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,The model recognized both New York programs but reduced PFL to an unsupported $49.31 and vacillated among obsolete rates and caps. PFL is $411.91 and disability contributions are $31.20. -us,scenario_082,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,The model incorrectly asserted that New York has no mandatory employee payroll tax. It omitted $411.91 of PFL and $31.20 of disability contributions. -us,scenario_082,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,The model included only federal Social Security and Medicare after incorrectly excluding mandatory New York employee payroll taxes. The omitted PFL and disability contributions total $443.11. -us,scenario_082,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an incorrect 0.388% PFL rate and an incorrect $89,343 wage cap, yielding $346.65 instead of $411.91. Its $31.20 disability contribution and federal FICA calculations were correct." -us,scenario_082,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,The model invented a combined $129.39 estimate for New York SDI and PFL after first treating them as zero because they were not household inputs. These are policy-determined mandatory contributions totaling $443.11. -us,scenario_082,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,The model included the $31.20 New York disability contribution but omitted the $411.91 New York Paid Family Leave contribution. -us,scenario_082,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model correctly identified all four components but approximated New York PFL at $333 and disability insurance at $31. The trace applies exact amounts of $411.91 and $31.20. -us,scenario_082,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model applied only the 7.65% federal FICA rate. It omitted the mandatory $411.91 New York PFL contribution and $31.20 disability contribution. -us,scenario_082,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly subtracted the $21,208 employer-sponsored insurance premium from FICA wages, reducing the base to $78,987. The trace applies Social Security, Medicare, PFL, and disability contributions to the full $100,195 of taxable wages." -us,scenario_082,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly treated the listed employer-sponsored insurance premium as a pre-tax reduction of FICA wages and taxed only $78,987. Payroll taxes use the full $100,195 wage base, with exact New York contributions of $411.91 and $31.20." -us,scenario_082,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly reduced payroll-tax wages to $78,987 by subtracting employer-sponsored insurance premiums. The computation uses the full $100,195 wage base and exact New York PFL and disability contributions." -us,scenario_082,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"The model used an incorrect 0.388% New York PFL rate and an estimated $96,200 cap, producing $373.26. The applicable PFL contribution is $411.91; the other components were calculated correctly." -us,scenario_082,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly excluded all mandatory New York employee payroll taxes and also rounded federal FICA below its exact $7,664.92 amount. New York PFL and disability contributions add $443.11." -us,scenario_082,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,The model rounded federal FICA inaccurately and stopped there. It omitted $411.91 of New York PFL and $31.20 of disability contributions. -us,scenario_082,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly computed $7,664.92 of federal FICA but estimated combined New York contributions at $385.73. The exact PFL and disability amounts total $443.11." -us,scenario_082,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,The model included New York's $31.20 disability contribution but omitted the $411.91 Paid Family Leave contribution. -us,scenario_082,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,The model computed federal Social Security and Medicare only. It omitted mandatory New York PFL and disability contributions totaling $443.11. -us,scenario_082,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The model included only federal Social Security and Medicare and misstated even their exact combined amount, which is $7,664.92. It also omitted $443.11 of mandatory New York PFL and disability contributions." -us,scenario_082,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model used only the 7.65% federal FICA shortcut. It omitted $411.91 of New York PFL and $31.20 of disability contributions. -us,scenario_082,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model improperly subtracted $21,208 of employer-sponsored insurance premiums from FICA wages and then excluded New York payroll contributions. The computation taxes the full $100,195 wage base and adds $443.11 of PFL and disability contributions." -us,scenario_082,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly subtracted both the $21,208 insurance premium and the $181 traditional 401(k) contribution from FICA wages. Neither reduction applies to the traced payroll-tax base, and the model also omitted $443.11 of New York PFL and disability contributions." +us,scenario_082,local_income_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,The model imposed NYC resident income tax solely because the household lives in New York State. It should not have applied NYC rates without a listed NYC location; the separate local-income-tax liability is zero. +us,scenario_082,local_income_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model estimated NYC personal income tax from adjusted gross income even though the prompt specifies only New York State and treats unlisted location facts as false. With no NYC residence or other taxable locality listed, local income tax is zero." +us,scenario_082,local_income_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"The model calculated NYC head-of-household tax brackets on $79,254 of taxable income without any fact establishing NYC residence. New York State residence alone triggers no local income tax in this benchmark, yielding zero." +us,scenario_082,local_income_tax,glm-5.2,llm_error,state_local_rule,False,"The model explicitly assumed NYC residence because NYC is a New York locality with an income tax, contradicting the instruction that unlisted household facts are false. Its detailed NYC taxable-income, bracket, school-credit, and dependent-care-credit computation is inapplicable because no NYC location was provided." +us,scenario_082,local_income_tax,inkling,llm_error,state_local_rule,False,"The model treated the household as an NYC resident and applied NYC brackets and credit rules despite the prompt listing only New York State. Without a specified NYC residence, none of those local rules applies and the local-income-tax output is zero." +us,scenario_082,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model explicitly excluded New York PFL and disability contributions. Those mandatory employee payroll taxes add $411.91 and $31.20 to its otherwise correct $7,664.92 federal FICA calculation." +us,scenario_082,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an estimated 0.62% New York PFL rate and then inserted unspecified additional state withholding to reach its answer. The applicable New York components are $411.91 of PFL and $31.20 of disability contributions, not $640 of estimated state taxes." +us,scenario_082,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,The model recognized both New York programs but replaced the PFL contribution with an internally inconsistent estimate of about $49.31 and vacillated among obsolete caps. The required state additions are $411.91 for PFL and $31.20 for disability benefits. +us,scenario_082,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,The model incorrectly stated that New York has no mandatory employee payroll tax. It omitted $411.91 of PFL and $31.20 of disability contributions from the correctly computed federal FICA amount. +us,scenario_082,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,The model treated federal FICA as the entire payroll-tax liability and incorrectly excluded mandatory New York employee payroll taxes. It omitted $443.11 comprising $411.91 of PFL and $31.20 of disability contributions. +us,scenario_082,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an obsolete 0.388% PFL rate and an estimated wage cap, yielding only $346.65 of PFL. The trace applies $411.91 of PFL on $100,195, alongside the correctly included $31.20 disability contribution." +us,scenario_082,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,The model first treated New York contributions as zero and then appended an unsupported $129.39 estimate. The mandatory New York additions total $443.11: $411.91 of PFL plus $31.20 of disability contributions. +us,scenario_082,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,The model included the $31.20 New York disability contribution but omitted the $411.91 New York PFL contribution. Its federal Social Security and Medicare calculations were otherwise aligned with the trace. +us,scenario_082,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model understated New York PFL as approximately $333 instead of $411.91 and rounded the disability contribution to $31 instead of $31.20. The exact state additions raise federal FICA of $7,664.92 to $8,108.03." +us,scenario_082,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,The model applied only the combined 7.65% federal FICA rate and rounded the result. It omitted $411.91 of New York PFL and $31.20 of New York disability contributions. +us,scenario_082,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly subtracted the $21,208 employer-sponsored-insurance premium from the benchmark's $100,195 payroll-tax wage base. It also understated PFL: the trace taxes the full wages and adds $411.91 of PFL plus $31.20 of disability contributions." +us,scenario_082,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly reduced Social Security and Medicare wages from $100,195 to $78,987 by subtracting the listed insurance premium. Federal FICA is computed on $100,195, and the state portion is exactly $411.91 of PFL plus $31.20 of disability contributions." +us,scenario_082,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model incorrectly used wages net of the $21,208 insurance premium instead of the $100,195 wage input for FICA. It also failed to compute the state contributions precisely as $411.91 of PFL and $31.20 of disability benefits." +us,scenario_082,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"The model applied an obsolete 0.388% PFL rate and a fabricated $96,200 cap, producing $373.26. The correct PFL contribution is $411.91 on the $100,195 wage base, with the separately calculated $31.20 disability contribution." +us,scenario_082,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model incorrectly asserted that New York imposes no mandatory employee payroll tax and also rounded federal FICA below its exact $7,664.92 value. It omitted $411.91 of PFL and $31.20 of disability contributions." +us,scenario_082,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model reduced the calculation to a rounded federal FICA estimate and omitted the mandatory New York components. Exact federal FICA is $7,664.92, followed by $411.91 of PFL and $31.20 of disability contributions." +us,scenario_082,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly calculated federal FICA but estimated the combined New York PFL and disability contributions at $385.73. Their exact amounts are $411.91 and $31.20, totaling $443.11." +us,scenario_082,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,The model included New York's $31.20 disability contribution but omitted the $411.91 PFL contribution. Adding PFL to its federal FICA and disability components yields the required total. +us,scenario_082,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,The model computed only the 7.65% federal Social Security and Medicare taxes. It omitted mandatory New York PFL of $411.91 and disability contributions of $31.20. +us,scenario_082,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model limited payroll tax to Social Security and Medicare and also miscomputed or over-rounded those taxes below their exact $7,664.92 total. It omitted $443.11 of mandatory New York PFL and disability contributions." +us,scenario_082,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,The model used only the combined 7.65% federal FICA rate and rounded the result. It omitted the $411.91 New York PFL contribution and the $31.20 New York disability contribution. +us,scenario_082,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model improperly subtracted $21,208 of employer-sponsored-insurance premiums from FICA wages and then excluded all New York employee payroll taxes. The trace uses the full $100,195 for federal FICA and adds $411.91 of PFL and $31.20 of disability contributions." +us,scenario_082,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages by both the $21,208 insurance premium and the $181 traditional 401(k) contribution. The trace uses $100,195 for federal FICA and also includes $411.91 of New York PFL and $31.20 of disability contributions." +us,scenario_082,payroll_tax,inkling,llm_error,thresholds_rates,False,"The model correctly identified every component but estimated PFL at about $365 instead of $411.91 and rounded the federal and disability amounts. The exact New York state total is $443.11, bringing exact federal FICA of $7,664.92 to $8,108.03." us,scenario_082,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required output contract." -us,scenario_082,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model correctly derived federal FICA and rounded it to $7,665, but omitted $411.91 of New York PFL and $31.20 of disability contributions." -us,scenario_082,payroll_tax,qwen-3.7-max,llm_error,other,False,The model repeatedly produced contradictory New York estimates and then inserted a negative $200 state adjustment with no valid payroll-tax rule. The correct state additions are positive: $411.91 of PFL and $31.20 of disability contributions. +us,scenario_082,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,The model calculated and rounded federal Social Security and Medicare taxes but stopped there. It omitted $411.91 of mandatory New York PFL and $31.20 of disability contributions. +us,scenario_082,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model produced multiple contradictory New York estimates and then subtracted an invented negative $200 state adjustment to force its final value. New York instead adds $411.91 of PFL and $31.20 of disability contributions to $7,664.92 of federal FICA." +us,scenario_082,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated that no state payroll tax was included, yet its submitted $7,752.42 does not equal its own Social Security and Medicare components, which sum to $7,664.92. It also omitted the actual $411.91 PFL and $31.20 disability contributions." us,scenario_082,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_082,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model computed an overstated regular tax and then improperly subtracted an estimated $396 child and dependent care credit. The traced calculation instead produces $5,442.17 of regular tax and adds $156.38 of supplemental tax, with no such reduction in this output." -us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model substituted federal taxable income of $62,149 for New York taxable income. New York starts from AGI of $117,585.15 and applies the $11,200 state standard deduction and $1,000 exemption, yielding $105,385.15 before the regular and supplemental taxes." -us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model treated the filer as single and used the $8,000 single standard deduction despite the household qualifying for head-of-household treatment. It also introduced unsupported credit reductions instead of applying the head-of-household schedule and $156.38 supplemental tax to the traced $105,385.15 taxable income." -us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model used single filing status, the $8,000 single deduction, and a single-filer bracket formula. The correct pathway uses head-of-household status, an $11,200 standard deduction, a $1,000 dependent exemption, and the supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an approximately $8,000 deduction even while describing the filer as head of household. The applicable New York standard deduction is $11,200, followed by the $1,000 dependent exemption and the regular-plus-supplemental tax computation." -us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model calculated the return as single, itemized $9,917, and subtracted a $120 child-care credit. The engine uses head-of-household status and the larger $11,200 standard deduction, then adds $156.38 of supplemental tax to $5,442.17 of regular tax." -us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"After constructing the wrong single-filer tax base and schedule, the model reduced its computed tax to $4,800 using unspecified 'minor credits' that its own reasoning did not calculate. The applicable computation uses head-of-household taxable income of $105,385.15 and adds the $156.38 supplemental tax without that unsupported reduction." -us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted $20,691 and reduced taxable income to $95,971, even though the allowable New York deduction is the larger-of comparison won by the $11,200 standard deduction, plus the $1,000 exemption. This understated the tax base and also omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $10,500 deduction instead of the $11,200 head-of-household standard deduction and then subtracted a $30 household credit. It also failed to apply the $156.38 supplemental tax that raises the $5,442.17 regular tax to $5,598.55." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer reflects only a rough wage-based bracket estimate and does not account for the traced New York AGI, dependent exemption, or supplemental tax. The required base is $105,385.15, producing $5,442.17 of regular tax plus $156.38." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model reduced New York taxable income to $84,178 instead of $105,385.15. It therefore removed $21,207.15 too much from the tax base and never reached the correct regular and supplemental tax calculation." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model began from New York AGI of $96,454 rather than $117,585.15, causing taxable income to be understated by more than $26,000. The correct deductions leave $105,385.15 taxable, after which both regular and supplemental tax apply." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The unexplained estimate does not implement the traced head-of-household schedule and supplemental-tax step. Taxable income of $105,385.15 yields $5,442.17 of regular tax plus $156.38, not $4,220." -us,scenario_082,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, violating the required output contract." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model stopped at a rounded bracket estimate and omitted the exact New York supplemental-tax computation. The regular tax is $5,442.17 and the required $156.38 supplement brings the output to $5,598.55." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model's $7,777 estimate is incompatible with the tax on the traced $105,385.15 taxable income and gives no bracket arithmetic supporting the excess. The applicable regular and supplemental taxes total $5,598.55." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model closely reconstructed taxable income but applied the New York tax schedule incorrectly and omitted the $156.38 supplemental tax. On $105,385.15, the regular tax is $5,442.17 and the supplement produces $5,598.55." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model named the correct filing status and deduction concept but produced a liability far below the tax generated by $105,385.15 of taxable income. It failed to apply the full graduated schedule and the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used approximately the right taxable-income scale but applied the head-of-household rates incorrectly. The schedule yields $5,442.17 of regular tax, after which the $156.38 supplemental tax produces $5,598.55." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model mentioned the $11,200 standard deduction but omitted the $1,000 dependent exemption from its stated tax-base construction. It also failed to separate the correct $5,442.17 regular tax from the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave a rounded estimate without performing the supplemental-tax step. The traced schedule produces $5,442.17 of regular tax and requires adding $156.38." -us,scenario_082,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $96,454 instead of $117,585.15 and consequently understated taxable income as $83,354 instead of $105,385.15. It also subtracted a $120 child-care credit rather than applying the traced regular tax plus supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, violating the required output contract." -us,scenario_082,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model reconstructed taxable income within about $9 but used the wrong head-of-household rate calculation and omitted the distinct supplemental-tax formula. The correct components are $5,442.17 of regular tax and $156.38 of supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model allowed the full $33,000 long-term capital loss to erase income instead of limiting the net capital-loss deduction in AGI to $3,000. It also invoked child-related credits to reduce the liability to zero even though the traced before-refundable-credits computation remains $5,598.55." -us,scenario_082,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the entire $30,711 net capital loss when computing federal AGI instead of applying the $3,000 annual capital-loss limit, understating AGI by $27,711. It then used single filing status and produced internally inconsistent arithmetic by increasing tax after mentioning a household credit." -us,scenario_082,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model substituted a flat 33% child-credit calculation for the $1,000 New York base reduced by the $709.50 AGI phaseout, and it estimated the dependent-care credit as $150 instead of applying the 60% New York factor to the $600 federal credit. The correct components are $290.50 and $360." -us,scenario_082,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that New York lacks refundable child and dependent-care credits and excluded both credits solely because of income. This filer receives a phased-down $290.50 New York Child Tax Credit and a $360 refundable New York Child and Dependent Care Credit. -us,scenario_082,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model replaced the child credit's $1,000 base and $709.50 phaseout with an unsupported $110 amount, then incorrectly treated the dependent-care credit as nonrefundable at this AGI. The refundable dependent-care credit is $360, producing $650.50 with the $290.50 child credit." -us,scenario_082,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model treated both New York credits as fully eliminated at this income. The child credit is only reduced from $1,000 to $290.50, and the refundable dependent-care calculation still yields $360." -us,scenario_082,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,The model incorrectly treated the income thresholds as eliminating the New York Child Tax Credit and Child and Dependent Care Credit. The applicable calculations leave $290.50 of child credit and $360 of dependent-care credit. -us,scenario_082,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model failed to apply the $709.50 phaseout to the $1,000 New York child-credit base, instead using 33% of a $2,000 federal CTC, and it used a 20% New York dependent-care factor instead of 60%. Those errors produce $660 and $120 rather than the correct $290.50 and $360." -us,scenario_082,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model incorrectly concluded that income fully phases out the New York child credit and overlooked the refundable dependent-care credit. At AGI $117,585.15, the child-credit phaseout leaves $290.50 and the dependent-care formula yields $360." -us,scenario_082,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used a 20% New York factor on the $600 federal dependent-care credit instead of the applicable 60% factor, understating that credit as $120 rather than $360. It also omitted the $290.50 phased-down New York Child Tax Credit." -us,scenario_082,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model used $330 instead of applying the $709.50 phaseout to the $1,000 child-credit base, and estimated the dependent-care credit as $300 rather than $360. It also added a $63 NYC School Tax Credit that is not part of this household's refundable state-credit calculation." -us,scenario_082,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model assigned a flat $330 child credit instead of reducing the $1,000 base by the $709.50 AGI phaseout to $290.50. It entirely omitted the $360 refundable New York Child and Dependent Care Credit." -us,scenario_082,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model used a $100 minimum child credit instead of the $290.50 amount remaining after the $709.50 phaseout, and used a 20% rather than 60% New York factor for dependent care. The correct dependent-care credit is $360, not $120." -us,scenario_082,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a 110% New York dependent-care factor instead of the 60% factor applicable at AGI $117,585.15, producing $660 rather than $360. It also used a flat $330 child credit instead of the $290.50 phased-down amount." -us,scenario_082,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model used $330 for the child credit instead of applying the $709.50 phaseout to its $1,000 base, which leaves $290.50. It also omitted the $360 refundable dependent-care credit." -us,scenario_082,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, so the required output was missing." -us,scenario_082,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model overlooked both refundable New York credits indicated by the household facts. The child and childcare inputs generate a $290.50 New York Child Tax Credit and a $360 New York Child and Dependent Care Credit. -us,scenario_082,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model wrongly required explicit credit-specific eligibility inputs rather than deriving eligibility from the listed child, income, and childcare facts. Those facts produce refundable credits of $290.50 for the child credit and $360 for dependent care." -us,scenario_082,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a 20% New York factor on the $600 federal dependent-care credit instead of the applicable 60%, yielding $120 rather than $360. It also incorrectly eliminated the child credit, whose income phaseout leaves $290.50." -us,scenario_082,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model took the $1,000 New York child-credit base as the final refundable amount and failed to subtract the $709.50 AGI phaseout. It also omitted the $360 dependent-care credit, so the correct components are $290.50 and $360." -us,scenario_082,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly derived the $290.50 child credit but applied a 20% New York factor to the $600 federal dependent-care credit. The applicable factor is 60%, making the dependent-care credit $360 rather than $120." -us,scenario_082,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model incorrectly excluded the one-year-old child from the New York child-credit pathway and treated income as eliminating all refundable credits. The child qualifies for a $1,000 base credit reduced to $290.50, and the childcare expenses generate another $360." -us,scenario_082,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model failed to recognize the qualifying-child and dependent-care pathways. They generate refundable New York credits of $290.50 and $360, respectively." -us,scenario_082,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model incorrectly treated the federal CTC as fully phased out and then used that conclusion to eliminate the New York child credit, while also omitting the dependent-care credit. New York's calculation leaves $290.50 after its own phaseout, and the dependent-care calculation adds $360." -us,scenario_082,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model considered only the state EITC pathway and incorrectly concluded that no other refundable credits qualified. The household receives a $290.50 New York Child Tax Credit and a $360 New York Child and Dependent Care Credit independently of EITC eligibility. -us,scenario_082,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, so the required output was missing." -us,scenario_082,state_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model imposed a $100 minimum after treating the expanded child credit as fully phased out, rather than subtracting the actual $709.50 phaseout from the $1,000 base to obtain $290.50. It also used a 20% rather than 60% New York dependent-care factor, producing $120 instead of $360." -us,scenario_082,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly tied refundable-credit availability to state tax liability and treated income as exceeding the credits' range. Refundability does not require remaining pre-credit liability here, and the formulas produce $290.50 of child credit plus $360 of dependent-care credit." -us,scenario_082,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model considered the New York earned income credit but omitted the separate child and dependent-care credit pathways. Those pathways yield $290.50 and $360 even though income is too high for the EITC. -us,scenario_083,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an artificially low post-TCJA-expiration single standard deduction of about $8,328, creating $2,682 of taxable income from the $11,010 AGI. PolicyEngine's 2026 single filer standard deduction is above $11,010, so taxable income is zero and the 10% bracket calculation never applies." -us,scenario_083,federal_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The reasoning correctly determined that the age-20 filer fails the childless EITC age requirement and qualifies for no other refundable credit, but the submitted $540 contradicts that derivation. It inserted an EITC-sized amount after explicitly deriving total refundable credits of zero." -us,scenario_083,federal_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,The reasoning correctly excluded the EITC because the childless filer is only 20 and excluded the refundable CTC because there are no qualifying children. The submitted $538 nevertheless adds a refundable credit that its own eligibility analysis established does not exist. -us,scenario_083,federal_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model explicitly derived EITC eligibility of zero under the childless-filer age rule and found no other refundable credits, then submitted $649. The final value contradicts the model's correct categorical-eligibility analysis." -us,scenario_083,federal_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,The model replaced the applicable minimum age of 25 for a childless EITC claimant with an age-over-19 rule. It therefore applied the 7.65% EITC phase-in to wages even though this 20-year-old is categorically ineligible. -us,scenario_083,federal_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model used an incorrect age threshold of 18 for the childless EITC instead of the applicable minimum age of 25. Its $539.33 is exactly the ineligible 7.65% phase-in credit on $7,050 of earnings." -us,scenario_083,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model treated low earnings as sufficient for the no-child EITC and ignored the childless claimant age requirement. At age 20, the filer is ineligible, so no maximum or partial EITC enters refundable credits." -us,scenario_083,federal_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model calculated and phased out a childless EITC without first applying the minimum-age eligibility test. Because the filer is 20, both its $539 phase-in amount and $24 phase-out calculation are inapplicable." -us,scenario_083,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model calculated a no-child EITC from earnings while omitting the childless claimant age test. The age-20 filer is ineligible regardless of being below an income threshold, leaving no refundable credit." -us,scenario_083,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly applied ACA Medicaid expansion to Texas and treated the 138% FPL adult MAGI threshold as available there. It also invoked asset limits as if low assets could establish eligibility, but the decisive issue is that this 20-year-old single adult is in no Texas Medicaid category at all." -us,scenario_083,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented an income-based Texas adult Medicaid pathway for a low-income young adult. PolicyEngine's Texas rules assign this person to no Medicaid category, so income below MAGI thresholds does not create eligibility for this non-disabled, non-SSI, non-dependent 20-year-old." -us,scenario_083,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model identified the correct wage-only FICA pathway but then contradicted its own arithmetic and submitted a fabricated rounded value. Its stated calculation gives $437.10 of Social Security and $102.22 of Medicare, totaling $539.33, while its final $540.33 adds an extra dollar not produced by any payroll-tax component." -us,scenario_083,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model used the correct wage base and correctly excluded Additional Medicare Tax and Texas payroll tax, but miscomputed the combined FICA amount. Applying 7.65% to $7,050 yields $539.33, not $540.90, so its answer reflects an arithmetic error in applying the Social Security and Medicare rates." -us,scenario_083,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model stated the correct employee FICA formula and nearly the correct subtotal, then replaced it with an unsupported netted figure of $498.95. There is no employee-side Texas payroll-tax adjustment or netting step that reduces Social Security plus Medicare, so the correct wage-only FICA liability remains $539.33." -us,scenario_083,payroll_tax,minimax-m3,llm_error,other,False,"The model computed the correct components and even stated the correct total of $539.33, then rounded it to $538 for reporting. Payroll tax is reported as an annual dollar amount with cents in this benchmark, and rounding $539.33 to $538 is an invalid final reporting step." -us,scenario_083,self_employment_tax,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly classified the $3,960 of rental income as self-employment income and applied the 92.35% earnings adjustment and 15.3% SECA rate to it. Under the benchmark facts, rental income is not net earnings from self-employment, so there are no self-employment earnings above the $400 filing threshold and no self-employment tax." -us,scenario_083,snap,claude-opus-4.7,llm_error,asset_resource,False,"The model skipped the Texas SNAP resource test and proceeded directly from income eligibility to the allotment formula. The $6,620 bank balance exceeds the applicable liquid-resource limit, making the household ineligible and its annual benefit $0." -us,scenario_083,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model explicitly treated Texas broad-based categorical eligibility as waiving the asset test. Texas retains an applicable liquid-resource limit, which the household's $6,620 bank balance exceeds, so no allotment calculation applies." -us,scenario_083,snap,claude-opus-5,llm_error,asset_resource,False,"The model used only the gross-income test, earned-income deduction, standard deduction, and allotment formula. It omitted the Texas liquid-resource test, under which $6,620 in bank assets makes this one-person household ineligible." -us,scenario_083,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model failed to apply the Texas liquid-resource limit and instead calculated a benefit from net income. It also invented a shelter deduction from the mortgage balance despite no mortgage payment or other shelter expense being reported, but the prior dispositive error is that $6,620 in bank assets makes the household ineligible." -us,scenario_083,snap,gemini-3-flash-preview,llm_error,asset_resource,False,"The model went straight from monthly income deductions to the maximum-allotment formula without screening resources. The reported $6,620 bank balance exceeds Texas's applicable SNAP liquid-resource limit, so the benefit is $0." -us,scenario_083,snap,glm-5.2,llm_error,asset_resource,False,"The model subtracted income deductions and 30% of net income from an estimated maximum allotment but omitted resource eligibility. Because the household has $6,620 in bank assets above the Texas liquid-resource limit, it receives no SNAP benefit." -us,scenario_083,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model calculated countable income and an allotment without applying the Texas SNAP resource test. The household's $6,620 bank balance exceeds the applicable limit, terminating eligibility before the $298-minus-30%-of-income calculation." -us,scenario_083,snap,grok-4.5,llm_error,asset_resource,False,"The model explicitly asserted that Texas BBCE waives assets and therefore ignored the $6,620 bank balance. Texas retains an applicable liquid-resource limit, which this household exceeds, so the correct benefit is $0." -us,scenario_083,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model omitted the Texas liquid-resource eligibility test and applied the allotment formula despite $6,620 in bank assets above the applicable limit. Its annual deduction and benefit arithmetic is also internally inconsistent, but resource ineligibility alone fixes the benefit at $0." -us,scenario_084,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model converted disability, blindness, SSDI receipt, and surviving-spouse status into categorical Medicaid eligibility even after recognizing that income and assets exceed SSI-related limits. North Carolina Medicaid eligibility still requires a qualifying coverage pathway; PolicyEngine found medicaid_category NONE, and immigration eligibility alone does not create Medicaid eligibility." -us,scenario_084,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model's explanation correctly identified that income is above the aged/blind/disabled income limit and that there is no MAGI child or parent category, but it submitted eligible anyway. It treated the disability pathway as available despite its own reasoning showing the disabled adult income test was failed; PolicyEngine therefore assigns no Medicaid category." -us,scenario_084,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented spend-down eligibility from unreimbursed medical expenses and treated that as a Medicaid-qualifying pathway for this case. PolicyEngine's North Carolina Medicaid calculation did not place the head in an aged/blind/disabled or medically needy eligibility category, and the trace assigns medicaid_category NONE rather than applying medical-expense spend-down to create eligibility." -us,scenario_084,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed PolicyEngine treats disabled and blind adults as Medicaid eligible under disability-related pathways without applying income, resource, SSI, or category tests. In the reference trace, the head receives no SSI and qualifies through no Medicaid category, so disability and blindness alone do not establish eligibility." -us,scenario_084,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model excluded SSDI from MAGI and computed expansion MAGI from only taxable interest and disability benefits, placing the head below 138% FPL. PolicyEngine's MAGI for Medicaid is 1.56 FPL, above the adult expansion threshold, and no asset-free expansion pathway applies once the MAGI category fails." -us,scenario_084,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted full-year disability status into satisfaction of Medicare's 24-month SSDI entitlement waiting period. A constant status during 2026 establishes only that year, so the age-62 head has no established Medicare disability pathway." -us,scenario_084,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model treated disability and receipt of Social Security disability income as sufficient proof that the 24-month Medicare waiting period had elapsed. Neither fact supplies the required prior entitlement duration, and the head is under age 65." -us,scenario_084,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model incorrectly inferred completion of the 24-month SSDI waiting period from disability being constant throughout the tax year. That instruction does not establish SSDI entitlement before 2026, leaving the 62-year-old ineligible." -us,scenario_084,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model explicitly assumed established disability-benefit receipt and therefore supplied an unlisted 24-month SSDI history. The prompt requires unlisted statuses and facts to be false, so current SSDI income does not establish Medicare eligibility before age 65." -us,scenario_084,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,The model treated current Social Security disability benefits as immediate Medicare qualification. It omitted the required 24-month disability-entitlement waiting period and the head is only 62. -us,scenario_084,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model cited the 24-month waiting period but treated current SSDI receipt as proof that it was completed. No prior entitlement duration is listed, so the under-65 disability pathway is not established." -us,scenario_084,head_medicare_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model expressly inferred at least 24 months of SSDI receipt from receiving SSDI at age 62. Age and current receipt do not establish duration, and the prompt forbids inferring that unlisted history." -us,scenario_084,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,The model treated disability plus current Social Security disability income as sufficient for Medicare. It omitted the 24-month SSDI entitlement requirement for an individual under age 65. -us,scenario_084,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model equated receipt of Social Security disability income with Medicare qualification. Current receipt does not establish completion of the required 24-month entitlement period, and the head is under 65." -us,scenario_084,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,The model treated SSDI beneficiary status as automatically conferring Medicare eligibility. It failed to require 24 months of disability entitlement before Medicare eligibility for this 62-year-old. -us,scenario_084,head_medicare_eligible,gemini-3.6-flash,llm_error,age_disability,False,"The model made disability and current SSDI income sufficient for Medicare eligibility. The under-65 pathway additionally requires the specified qualifying entitlement history, which is not listed." -us,scenario_084,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,The model mischaracterized PolicyEngine's rule as granting Medicare eligibility to any disabled SSDI recipient under 65. The facts do not establish the required 24-month SSDI entitlement period or another qualifying medical pathway. -us,scenario_084,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model treated disability and surviving-spouse status as indicators of Medicare eligibility at age 62. Surviving-spouse status is not an under-65 Medicare pathway, and no completed disability waiting period or qualifying medical condition is given." -us,scenario_084,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,The model asserted that PolicyEngine converts disabled status and current SSDI income into Medicare eligibility. Those inputs do not establish the required 24-month entitlement history for a person under 65. -us,scenario_084,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,The model treated disability and current Social Security disability income as sufficient support for the under-65 pathway. It omitted the required prior SSDI entitlement duration and supplied no other qualifying Medicare condition. -us,scenario_084,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model incorrectly made receipt of Social Security disability income immediately qualifying for Medicare. At age 62, eligibility requires a completed 24-month entitlement period or another listed qualifying medical pathway." -us,scenario_084,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,The model treated disabled status and SSDI income as the complete disability pathway. It failed to apply the 24-month SSDI entitlement requirement for Medicare before age 65. -us,scenario_084,head_medicare_eligible,grok-4.3,llm_error,age_disability,False,The model asserted that SSDI had been received for a sufficient duration even though no duration was provided. The required 24-month entitlement history cannot be inferred from current-year receipt. -us,scenario_084,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,"The model declared the 24-month SSDI waiting period satisfied solely from current disability and SSDI income. The prompt lists no prior entitlement duration, so that under-65 Medicare condition is not met." -us,scenario_084,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,The model treated being disabled and receiving SSDI as automatically qualifying before age 65. It omitted the required 24-month disability-entitlement waiting period. -us,scenario_084,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model explicitly assumed that the 24-month waiting period was satisfied from the amounts of disability benefits and Social Security disability income. Benefit amounts reveal no entitlement duration, and the prompt forbids supplying that unlisted history." -us,scenario_084,head_medicare_eligible,qwen-3.7-max,llm_error,age_disability,False,The model incorrectly used constant status throughout the tax-benefit year to satisfy a 24-month SSDI waiting period. Full-year 2026 receipt establishes at most 12 months and does not provide the unlisted prior entitlement needed for Medicare at age 62. -us,scenario_084,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly recognized that an elderly/disabled one-person SNAP household uses the net income test, but it failed to apply the one-person net income limit as an eligibility cutoff after computing net income around $1,738 per month. It then contradicted its own formula showing a zero benefit and invented a $183 monthly benefit by applying a minimum-benefit concept to a household that is ineligible because its net income exceeds the SNAP limit." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived approximately $2,323 using $21,424 of taxable income and the 2026 brackets, then submitted $940 instead. The submitted value does not follow from any calculation in its explanation." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model included all Social Security in AGI and then invented usable elderly and Saver’s credits. Only $14,050.60 of Social Security is taxable, and this filer receives no nonrefundable credit that eliminates the $2,322.76 liability." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model understated the aged standard deduction component, using roughly $17,000 before the $6,000 senior deduction instead of the correct $18,150. This overstated taxable income by about $1,150 and produced $2,470 instead of $2,322.76." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at about $20,705 instead of $14,050.60 and used an understated base standard deduction. Correct AGI is $45,573 and the full deduction is $24,150, leaving $21,423 taxable." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly calculated taxable income near $21,424 and tax near $2,323, then submitted $1,170. Nothing in its reasoning supplies a credit or other adjustment that reduces the correctly computed tax to the submitted amount." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the $6,000 senior deduction and used only a roughly $17,000 aged standard deduction. It also improperly treated a desired traditional 401(k) contribution as an $8 adjustment despite zero wages; the trace instead has only $0.40 of above-the-line deductions." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model first omitted the $6,000 senior deduction and calculated tax of about $3,189, then submitted $626 without identifying any deduction or credit producing that reduction. Its final number is disconnected from its stated computation." -us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used obsolete-looking values: a $9,800 standard deduction and 10%/15% brackets. It also overstated taxable Social Security; the correct inputs are $45,573 of AGI, a $24,150 total deduction, and the 2026 10%/12% main-rate schedule on $21,423." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration restored a personal exemption, a small standard deduction, and a 15% second bracket. It omitted the current-law $6,000 senior deduction; the applicable total deduction is $24,150 and the second marginal rate on this income is 12%." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated roughly the entire $58,949 cash-income total as gross taxable income, rather than including only $14,050.60 of Social Security in AGI. It also failed to apply the full $24,150 deduction that reduces $45,573 of AGI to $21,423." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used projected pre-TCJA baseline deductions and a personal exemption instead of the applicable 2026 deductions. It omitted the $6,000 senior deduction, so it failed to reduce the correctly identified $45,573.60 AGI to $21,423 of taxable income." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied post-TCJA-expiration personal-exemption rules and a 15% bracket while omitting the $6,000 senior deduction. Current-law deductions total $24,150, and the income above the 10% bracket is taxed at 12%." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly computed taxable Social Security and AGI but subtracted only a $17,450 aged standard deduction. It omitted the $6,000 senior deduction; the total deduction is $24,150, producing $21,423 rather than $28,123.60 of taxable income." -us,scenario_085,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model correctly computed $45,573.60 of AGI but deducted only about $17,000. It omitted the $6,000 senior deduction and understated the remaining standard-deduction amount, overstating taxable income by roughly $7,150." -us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the absence of wages and dependents as eliminating federal income tax. Taxable retirement, interest, and Social Security income create $45,573 of AGI, and after $24,150 of deductions the remaining $21,423 generates $2,322.76 of tax with no offsetting nonrefundable credits." -us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model asserted that deductions, contributions, and credits offset the income without calculating them. The actual deductions reduce AGI only to $21,423 of taxable income, and no nonrefundable credit eliminates the resulting $2,322.76 liability." -us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The submitted $4,200 implies substantially more than the correct $21,423 of taxable income or the use of higher rates. The correct derivation applies the full $24,150 deduction to $45,573 of AGI before calculating tax under the 2026 main rates." -us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset personal exemption and 15% bracket, and its combined $15,889 deduction omitted the $6,000 senior deduction. The applicable deduction is $24,150 and the second bracket rate on this taxable income is 12%." -us,scenario_085,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model taxed about 85% of the entire Social Security benefit instead of applying the two-tier inclusion formula, which yields $14,050.60 of taxable benefits. It also used a $10,631 post-TCJA-reversion deduction and a 15% bracket instead of the $24,150 deduction and applicable 10%/12% rates." -us,scenario_085,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. It therefore failed the required structured-output contract before any substantive tax calculation could be evaluated. -us,scenario_085,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model included the maximum 85% of Social Security instead of the correct $14,050.60 taxable amount, then claimed substantial taxable income produced near-zero tax. Even under its own overstated-income calculation, the conclusion contradicts the progressive rate schedule; the correct $21,423 taxable income produces $2,322.76." -us,scenario_085,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model wrongly included the maximum 85% of Social Security rather than the $14,050.60 yielded by the statutory two-tier formula and omitted the $6,000 senior deduction. It then calculated tax near $4,500 but submitted $8,587.50 without any computation supporting the additional amount." +us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model incorrectly subtracted an estimated $396 New York child and dependent care credit from this output. The trace instead applies $5,442.17 of main tax plus $156.38 of supplemental tax, with no such reduction, yielding $5,598.55." +us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model substituted federal taxable income of $62,149 for New York taxable income. New York begins with $117,585.15 AGI and applies its own $11,200 deduction and $1,000 exemption, leaving $105,385.15 before the main and supplemental tax computations." +us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model treated the filer as single, used the $8,000 single deduction, and omitted the $1,000 dependent exemption even though the household qualifies as head of household. It also never reconciled its stated $6,221 pre-credit tax with its unexplained $5,076 final figure or applied the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model used single filing status and an $8,000 deduction instead of head-of-household status, the $11,200 deduction, and the $1,000 dependent exemption. It then reduced its computed $6,326 tax by an unspecified adjustment and omitted the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an approximately $8,000 deduction despite identifying head-of-household filing. The applicable deduction is $11,200, followed by a $1,000 exemption, and the resulting $5,442.17 main tax requires a further $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model itemized $9,917 instead of selecting the larger $11,200 New York standard deduction and also omitted the $1,000 dependent exemption. It further subtracted a $120 child-care credit and failed to add the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"After computing roughly $6,193 of tax, the model invented approximately $1,393 of unspecified household and child-related credit reductions to reach $4,800. The requested computation instead has $5,442.17 of main tax and a $156.38 supplemental tax, without those reductions." +us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted $20,691 of itemized expenses, driving taxable income down to $95,971. The allowable comparison selects the $11,200 standard deduction over $11,169.24 of itemized deductions, and the $1,000 exemption then leaves taxable income of $105,385.15." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used a $10,500 deduction and taxable income of $107,086 rather than the $11,200 deduction, $1,000 exemption, and $105,385.15 taxable income. It also subtracted a $30 household credit and omitted the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The answer used only a rough wage-based estimate and did not incorporate the traced AGI, deduction comparison, dependent exemption, or supplemental tax. The required steps yield $5,442.17 of main tax plus $156.38 of supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated New York taxable income as $84,178. The traced New York AGI is $117,585.15, and subtracting $11,200 and $1,000 leaves $105,385.15, which is also subject to the supplemental-tax computation." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated New York AGI as $96,454 and taxable income as $79,254. The income components produce $117,585.15 of AGI and $105,385.15 of taxable income after the $11,200 deduction and $1,000 exemption." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The answer is a generic bracket estimate that does not account for the traced $117,585.15 AGI, $105,385.15 taxable income, or $156.38 supplemental tax. Those steps produce $5,598.55, not $4,220." +us,scenario_082,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no output or explanation for the requested variable. +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model rounded from an unspecified taxable-income and credit estimate instead of completing the New York computation. The exact derivation is $5,442.17 of main tax plus $156.38 of supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $7,777 estimate applies neither the traced New York liability nor a coherent rate calculation to $105,385.15 of taxable income. The applicable computation produces $5,442.17 of main tax, followed by $156.38 of supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model derived taxable income within $9 of the traced $105,385.15 but converted it to only $5,548. It misapplied the New York tax computation and omitted the full result of $5,442.17 main tax plus $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's $2,954.80 figure substantially understates the tax produced by New York's head-of-household schedule on $105,385.15. It failed to calculate the $5,442.17 main tax and add the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used taxable income near the traced amount but applied the rate computation incorrectly, producing $5,815. On $105,385.15, the main tax is $5,442.17 and the supplemental tax is $156.38." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model mentioned only the $11,200 standard deduction and did not apply the $1,000 dependent exemption in deriving taxable income. The correct taxable income is $105,385.15, producing $5,442.17 of main tax plus $156.38 of supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model returned a rounded estimate without performing the required traced computation. The main tax is $5,442.17 and the separate supplemental tax is $156.38, so rounding to $5,500 loses a material calculation step." +us,scenario_082,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated federal and New York AGI as $96,454, causing taxable income to fall to $83,354. The trace uses $117,585.15 of AGI and $105,385.15 of taxable income; the model also wrongly subtracted a $120 child-care credit and omitted supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model claimed $2,000 of exemptions instead of the traced $1,000 exemption and then subtracted a $120 child-care credit. The applicable calculation uses taxable income of $105,385.15 and adds $156.38 of supplemental tax to $5,442.17 of main tax." +us,scenario_082,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no output or explanation for the requested variable. +us,scenario_082,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model closely reproduced the taxable-income inputs but applied a simplified bracket schedule to obtain $5,698.83. The New York computation on $105,385.15 produces $5,442.17 of main tax and then adds $156.38 of supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the full $33,000 long-term capital loss as offsetting income instead of applying the $3,000 annual net-capital-loss limitation embedded in AGI. It also invoked child-related credits without computing them, even though the traced taxable income remains $105,385.15 and generates positive tax." +us,scenario_082,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the entire $30,711 net capital loss in reaching $89,951 of federal AGI instead of limiting the annual capital-loss deduction to $3,000. It also used single filing status and produced a final amount that does not follow from its own stated $4,292.89 tax." +us,scenario_082,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model overstated federal AGI as $139,586, used the $8,000 single deduction, and introduced an unsupported $5,516 adjustment. It then subtracted household and child credits, including refundable credits that do not reduce this before-refundable-credits output, while omitting the traced supplemental-tax step." +us,scenario_082,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model applied an obsolete 33%-of-federal-CTC shortcut instead of starting the 2026 New York child credit at $1,000 and subtracting the $709.50 AGI phaseout. It also understated the dependent-care credit, which is $360 after applying New York’s 60% rate to the $600 federal CDCC." +us,scenario_082,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that New York lacks refundable child and dependent-care credits and treated the income as eliminating them. The child qualifies for a phased-down $290.50 New York child credit, and the reported childcare expenses generate a $360 refundable dependent-care credit." +us,scenario_082,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,The model incorrectly limited the age-one child to a $110 under-four amount and classified the New York dependent-care credit as nonrefundable at this AGI. The applicable rules produce a $290.50 phased-down child credit and a refundable $360 dependent-care credit. +us,scenario_082,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model treated the household’s income as fully eliminating both New York credits. The child-credit phaseout leaves $290.50, while the dependent-care formula still awards $360 at the 60% New York rate." +us,scenario_082,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,The model treated the income thresholds as complete disqualifiers for the New York child and dependent-care credits. The child-credit phaseout leaves $290.50 and the dependent-care schedule produces $360. +us,scenario_082,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model derived the child credit as 33% of a $2,000 federal CTC instead of applying the $1,000 New York base and $709.50 phaseout, overstating that component by $369.50. It also used a 20% New York dependent-care rate instead of the applicable 60% rate, understating that component from $360 to $120." +us,scenario_082,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated New York AGI as fully phasing out the child credit and omitted the dependent-care credit. The phaseout leaves $290.50 of child credit, and $3,000 of qualifying care expenses generate another $360." +us,scenario_082,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used a 20% New York percentage on the $600 federal CDCC rather than the applicable 60%, producing $120 instead of $360. It also omitted the $290.50 child credit remaining after phaseout." +us,scenario_082,state_refundable_credits,gemini-3-flash-preview,llm_error,other,False,The model invented a $63 NYC School Tax Credit without any fact establishing NYC residence and miscomputed both genuine components. The correct components are a $290.50 phased-down child credit and a $360 dependent-care credit. +us,scenario_082,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model used a generic $330 Empire State Child Credit amount instead of the $1,000 base less the $709.50 phaseout, which yields $290.50. It omitted the separate $360 refundable dependent-care credit." +us,scenario_082,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model reduced the child credit to a $100 minimum instead of calculating the $290.50 amount left after the AGI phaseout. It also applied a 20% New York rate to the $600 federal CDCC rather than the applicable 60%, yielding $120 instead of $360." +us,scenario_082,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a 110% dependent-care rate that belongs to a lower AGI band, rather than the applicable 60% rate, so that credit is $360 rather than $660. It also used a generic $330 child credit instead of the phased-down $290.50 amount." +us,scenario_082,state_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model used a generic $330 child-credit amount instead of applying the $709.50 phaseout to the $1,000 base, which leaves $290.50. It omitted the $360 refundable dependent-care credit." +us,scenario_082,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for this requested output. +us,scenario_082,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model omitted both refundable credits despite the qualifying age-one child and reported childcare expenses. Those facts produce a $290.50 phased-down child credit and a $360 dependent-care credit. +us,scenario_082,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model wrongly required explicit credit-specific eligibility inputs instead of deriving eligibility from the child, income, and childcare facts. Those inputs produce $290.50 of child credit and $360 of dependent-care credit." +us,scenario_082,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model applied a 20% New York rate to the $600 federal CDCC instead of the applicable 60%, understating that credit from $360 to $120. It also incorrectly treated the child credit as fully eliminated rather than leaving $290.50 after phaseout." +us,scenario_082,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model awarded the full $1,000 New York child-credit base without subtracting the $709.50 AGI phaseout, which leaves $290.50. It also omitted the separate $360 dependent-care credit." +us,scenario_082,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly identified the $290.50 child credit but applied a 20% New York rate to the $600 federal CDCC. The applicable New York rate is 60%, so the dependent-care component is $360 rather than $120." +us,scenario_082,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model wrongly excluded the age-one child from the New York child credit and treated income as eliminating every refundable credit. The child generates a $290.50 phased-down credit, and the childcare expenses generate a $360 dependent-care credit." +us,scenario_082,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model omitted the qualifying New York child and dependent-care credit pathways. Their computed amounts are $290.50 and $360, respectively." +us,scenario_082,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model incorrectly treated the federal CTC as fully phased out and treated that conclusion as eliminating the New York child credit. The New York calculation independently starts at $1,000 and leaves $290.50 after phaseout, while the refundable dependent-care calculation adds $360." +us,scenario_082,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model considered only the state EITC pathway and omitted New York’s refundable child and dependent-care credits. Those credits equal $290.50 after phaseout and $360 under the applicable care-credit rate. +us,scenario_082,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,The model incorrectly treated the New York dependent-care credit as merely offsetting tax liability rather than refundable and omitted the child credit. The refundable amounts are $360 for dependent care and $290.50 for the phased-down child credit. +us,scenario_082,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for this requested output. +us,scenario_082,state_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model replaced the child-credit phaseout calculation with a $100 minimum, but the $1,000 base less the $709.50 phaseout leaves $290.50. It also used a 20% New York dependent-care rate instead of 60%, producing $120 rather than $360." +us,scenario_082,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model wrongly treated income as eliminating the New York child credit and ignored the independent refundable dependent-care formula. The child-credit phaseout leaves $290.50, and the childcare expenses produce $360." +us,scenario_082,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model considered the New York EITC income limit but omitted the separate child and dependent-care pathways. Those pathways produce $290.50 and $360 even though the EITC is zero. +us,scenario_082,state_refundable_credits,qwen3.8-max,llm_error,other,False,"The model collapsed the relevant credits into an unsupported approximate $313 and did not apply the traced formulas. The child credit is $290.50 after its phaseout, and the distinct dependent-care credit is $360." +us,scenario_083,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly assumed the individual provisions of the Tax Cuts and Jobs Act expired for 2026 and used an $8,328 single-filer standard deduction. The applicable 2026 standard deduction exceeds $11,010, so taxable income and the resulting 10% bracket tax are both zero." +us,scenario_083,federal_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly determined that the age-20 childless filer fails the EITC age requirement and that no other refundable credit applies, but then submitted $540 instead of carrying the resulting $0 into the output." +us,scenario_083,federal_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly concluded that the filer is too young for the childless EITC and has no qualifying child for refundable CTC, but its $538 output contradicts its own all-zero component calculation." +us,scenario_083,federal_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model explicitly found the age-20 filer ineligible for the childless EITC and found no other refundable credits, then failed to propagate that eligibility result and submitted $649 rather than $0." +us,scenario_083,federal_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated age 20 and nonstudent status as sufficient for the childless EITC and applied the 7.65% phase-in rate to wages. For 2026, a childless claimant must be at least age 25, so the EITC is zero before any rate calculation." +us,scenario_083,federal_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model used an incorrect age-18 threshold for the childless EITC and therefore applied the 7.65% phase-in formula to $7,050 of wages. The applicable minimum age is 25, making the age-20 filer categorically ineligible regardless of earned income, AGI, or investment income." +us,scenario_083,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model awarded the maximum no-child EITC solely from the household's low earnings and omitted the childless claimant age test. At age 20, the filer fails the minimum-age requirement, so no EITC is available." +us,scenario_083,federal_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model calculated an EITC phase-in and phase-out without first applying the childless claimant age requirement. Because the filer is 20 rather than at least 25, the EITC is zero and the phase-in and phase-out computations never apply." +us,scenario_083,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model inferred a no-child EITC from earned income and the phase-out threshold while omitting the claimant's age. The age-20 filer is ineligible under the childless EITC minimum-age rule, so the refundable credit total is zero." +us,scenario_083,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated low earnings alone as establishing EITC eligibility and assigned $1,762.50 without applying the no-child age restriction. The filer is age 20 and has no qualifying children, so the EITC and total federal refundable credits are zero." +us,scenario_083,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model falsely treated Texas as an ACA Medicaid-expansion state and applied a 138% FPL adult pathway that Texas does not provide. Its asset-limit discussion is irrelevant because this nondependent 20-year-old first fails to qualify for any Texas Medicaid category. +us,scenario_083,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a Texas Medicaid pathway for low-income young adults. Age 20 and MAGI below the income limit do not establish eligibility without a covered category, and this person qualifies through none." +us,scenario_083,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The answer treated age, low income, and Texas residence as sufficient without applying Texas's categorical eligibility rules. The head is a nondependent adult with no qualifying pathway, so the engine assigns `medicaid_category` NONE." +us,scenario_083,head_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated low income and age 20 as sufficient for WIC without applying WIC's categorical restriction. A nonpregnant, nonpostpartum, nonbreastfeeding 20-year-old adult is outside every qualifying WIC category, so passing the income test does not confer eligibility." +us,scenario_083,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model made inconsistent arithmetic, first understating Social Security tax as $436.10 and then deriving a $538.33 subtotal before submitting the unrelated value $540.33. Applying 6.2% and 1.45% to $7,050 yields $437.10 and $102.22, totaling $539.33." +us,scenario_083,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model named the correct wage base and payroll-tax rates but did not perform the stated multiplication correctly. A combined 7.65% tax on $7,050 produces $539.325, which rounds to $539.33, not $540.90." +us,scenario_083,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's submitted $498.95 contradicts its own component calculation and its statement that both Social Security and Medicare apply. It also misstated $7,050 × 1.45% as $102.175; the correct components are $437.10 and $102.22, totaling $539.33." +us,scenario_083,payroll_tax,minimax-m3,llm_error,other,False,The model correctly derived $539.33 and then improperly changed it to $538 under the label of rounding. Ordinary currency rounding preserves $539.33; it does not reduce the result by $1.33. +us,scenario_083,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The submitted $649.62 is inconsistent with applying employee Social Security and Medicare taxes to the stated $7,050 wages. The correct 7.65% combined tax on that wage base is $539.33, so the answer used an inflated base or rate unsupported by the household facts." +us,scenario_083,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly classified the $3,960 of rental income as self-employment income and applied the 92.35% adjustment and 15.3% tax rate to it. Ordinary rental income is excluded from net earnings from self-employment, leaving no self-employment earnings subject to the $400 threshold and yielding $0 of self-employment tax." +us,scenario_083,snap,claude-opus-4.7,llm_error,asset_resource,False,"The model never applied Texas's $5,000 SNAP countable-resource limit to the listed $6,620 bank balance. That excess makes the household ineligible, so its subsequent earned-income deduction, standard deduction, and allotment calculation cannot produce a benefit." +us,scenario_083,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model explicitly treated Texas broad-based categorical eligibility as waiving the asset test. Texas instead imposes a $5,000 resource limit under that pathway, and the $6,620 bank balance disqualifies the household before benefit computation." +us,scenario_083,snap,claude-opus-5,llm_error,asset_resource,False,"The model calculated an allotment solely from gross and net income and omitted the resource-eligibility step. The household's $6,620 bank assets exceed Texas's $5,000 SNAP resource limit, yielding no eligibility and no benefit." +us,scenario_083,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model skipped Texas's $5,000 resource test and proceeded to estimate a positive income-based allotment. The listed $6,620 bank balance disqualifies the household, and the mortgage balance supplies neither an asset-test exception nor a reported shelter payment deduction." +us,scenario_083,snap,gemini-3-flash-preview,llm_error,asset_resource,False,"The model applied the earned-income and standard deductions directly to the allotment formula without testing countable resources. The $6,620 bank balance exceeds Texas's $5,000 SNAP resource limit, so the correct computation stops at ineligibility." +us,scenario_083,snap,glm-5.2,llm_error,asset_resource,False,"The model subtracted income deductions and 30% of net income from a maximum allotment but omitted resource eligibility. Texas's $5,000 limit is exceeded by the household's $6,620 bank assets, forcing the SNAP amount to zero." +us,scenario_083,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model treated countable monthly income as the only binding eligibility input and calculated a positive allotment. It failed to apply Texas's $5,000 resource limit to the $6,620 bank balance, which makes the household ineligible." +us,scenario_083,snap,grok-4.5,llm_error,asset_resource,False,"The model explicitly asserted that Texas broad-based categorical eligibility waives assets. Texas retains a $5,000 countable-resource limit under that pathway, so the household's $6,620 bank balance eliminates SNAP eligibility before the net-income formula." +us,scenario_083,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model moved from income deductions to an annual allotment without applying the resource test. Because $6,620 in bank assets exceeds Texas's $5,000 SNAP resource limit, the household is ineligible regardless of its income-based allotment calculation." +us,scenario_084,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model correctly identified that income and assets exceed the SSI-related limits, then overrode those limits by treating disability, blindness, SSDI receipt, and surviving-spouse status as automatic categorical eligibility. None of those facts creates a qualifying North Carolina Medicaid pathway without satisfying the applicable financial requirements." +us,scenario_084,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model’s own reasoning found that the head exceeds the aged/blind/disabled income limit and has no MAGI parent or child category, but it submitted Yes anyway. Those findings yield no qualifying Medicaid category and therefore an eligibility value of 0." +us,scenario_084,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented medically needy spend-down eligibility by subtracting the listed annual medical expenses from income. The trace assigns no Medicaid category, and those expenses do not establish a qualifying North Carolina spend-down pathway for this head under PolicyEngine’s rules." +us,scenario_084,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated being disabled and blind as sufficient for a disability-related Medicaid pathway while leaving the required income and category tests unresolved. The head qualifies through no Medicaid category, so those statuses alone yield no eligibility." +us,scenario_084,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model excluded Social Security disability income from Medicaid MAGI merely because it is nontaxable for ordinary income-tax purposes. Medicaid MAGI includes the applicable nontaxable Social Security amount, producing 1.56 times FPL rather than the model’s $8,405 estimate and placing the head above the expansion threshold." +us,scenario_084,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted current SSDI receipt and full-year disability status into completion of Medicare’s 24-month SSDI waiting period. Constancy during 2026 does not establish SSDI entitlement in the preceding 24 months, so the under-65 disability pathway was not satisfied." +us,scenario_084,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model treated disability and present SSDI receipt as sufficient for Medicare before age 65, despite naming the 24-month requirement. No SSDI entitlement start date or completed waiting period was listed, so it improperly inferred the missing duration." +us,scenario_084,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,The model assumed that constant disability status throughout the tax year proves 24 months of SSDI receipt. It establishes only the person’s 2026 status and does not satisfy the prior-duration requirement for under-65 Medicare eligibility. +us,scenario_084,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model expressly assumed established benefit receipt and therefore completion of the 24-month SSDI waiting period. The prompt lists only annual 2026 SSDI income and supplies no prior entitlement duration, so that assumption incorrectly activated the under-65 Medicare pathway." +us,scenario_084,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model treated disability plus Social Security disability benefits as automatically qualifying for Medicare. For a 62-year-old, current SSDI receipt alone does not establish completion of the disability-based Medicare waiting period." +us,scenario_084,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model cited the 24-month waiting period but then inferred it solely from current disability and SSDI receipt. Neither fact establishes 24 months of prior SSDI entitlement, leaving the head ineligible at age 62." +us,scenario_084,head_medicare_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model explicitly inferred at least 24 months of SSDI receipt from receiving SSDI at age 62. Age and current receipt contain no information about the entitlement start date, so the required waiting period was not established." +us,scenario_084,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"The model treated disability and SSDI receipt as automatically conferring Medicare eligibility. It omitted the required qualifying under-65 entitlement pathway, including the completed SSDI waiting period, which the household facts do not establish." +us,scenario_084,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model equated receiving Social Security disability income with Medicare eligibility. At age 62, SSDI income does not by itself prove completion of the 24-month Medicare waiting period." +us,scenario_084,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,The model assumed that being disabled and receiving SSDI immediately qualifies an under-65 person for Medicare. It failed to require evidence that the SSDI-based Medicare waiting period had been completed. +us,scenario_084,head_medicare_eligible,gemini-3.6-flash,llm_error,age_disability,False,The model made disability plus current SSDI receipt sufficient for Medicare before age 65. It omitted the completed waiting-period requirement and inferred a qualifying entitlement history that was not listed. +us,scenario_084,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,The model asserted that PolicyEngine’s under-65 rule treats disability and SSDI income as sufficient. It failed to distinguish current disability income from an established qualifying Medicare disability entitlement after the required waiting period. +us,scenario_084,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model treated disability and surviving-spouse status as evidence of Medicare eligibility at age 62. Surviving-spouse status does not establish Medicare entitlement, and the facts do not establish completion of the SSDI waiting period." +us,scenario_084,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,The model incorrectly asserted that PolicyEngine converts the disabled flag and current Social Security disability income into Medicare eligibility. Those inputs do not establish the required duration of SSDI entitlement for the under-65 pathway. +us,scenario_084,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,The model treated disability and SSDI income as sufficient support for Medicare eligibility despite the head being under 65. It failed to require a completed 24-month SSDI waiting period or another qualifying medical pathway. +us,scenario_084,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,The model inferred Medicare eligibility directly from receiving Social Security disability income. Current receipt does not prove the prior entitlement duration required for Medicare before age 65. +us,scenario_084,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model activated the disability pathway from the disabled flag and current SSDI income alone. The under-65 pathway requires qualifying entitlement after the waiting period, and no completed waiting period was supplied." +us,scenario_084,head_medicare_eligible,grok-4.3,llm_error,age_disability,False,The model asserted sufficient SSDI duration even though the prompt gives no entitlement start date or receipt history before 2026. It therefore invented completion of the 24-month waiting period. +us,scenario_084,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,The model stated that the head satisfied the 24-month SSDI waiting period without any fact establishing 24 months of entitlement. Present-year SSDI income and disability status do not supply that missing duration. +us,scenario_084,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,The model treated being disabled and receiving SSDI as categorically sufficient for Medicare under age 65. It omitted the requirement that the qualifying SSDI entitlement persist through the applicable waiting period. +us,scenario_084,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model explicitly assumed the 24-month waiting period was satisfied from the two listed annual disability-income amounts. Payment amounts do not establish an entitlement start date or 24 months of prior receipt, so the assumption incorrectly created Medicare eligibility." +us,scenario_084,head_medicare_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model interpreted constant disability status and SSDI receipt throughout 2026 as completion of a 24-month waiting period. Full-year constancy covers only 12 months and says nothing about prior years, so the disability-based Medicare pathway was not established." +us,scenario_084,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model calculated that 30% of monthly net income exceeded the $298 maximum allotment, correctly producing $0, then discarded that result and asserted an unsupported $183 monthly benefit. It misapplied the minimum-benefit rule, which cannot convert income ineligibility or a nonpositive allotment calculation into $2,196 annually." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its written derivation reaches approximately $2,323 using the correct taxable Social Security, $24,150 deduction, and tax brackets, but it submits $940 instead. The final value contradicts its own arithmetic and reflects an unsupported last-step substitution." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It treated all Social Security benefits as AGI and then invented usable elderly and Saver's Credits. At this income the elderly credit is eliminated under its statutory income limitations, and the tiny listed contributions do not create credits capable of erasing the $2,322.76 liability." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It understated the applicable deductions as roughly $23,000 instead of $24,150. Applying the $18,150 aged standard deduction plus the $6,000 additional senior deduction yields taxable income of $21,423, not $22,574." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It overstated taxable Social Security as about $20,705 instead of $14,050.60. The two-tier Social Security formula adds $4,500 to 85% of provisional income above $34,000, producing AGI of $45,573.60 and taxable income of $21,423 after the full $24,150 deduction." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its reasoning correctly computes approximately $2,323 from taxable income near $21,424 and explicitly finds no applicable credits, but it submits $1,170. The submitted number is an unsupported replacement of its own correctly derived tax." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It omitted the $6,000 additional senior deduction and used only a $17,000 aged standard deduction. It also improperly reduced AGI by a desired traditional 401(k) contribution despite zero wages; the trace instead applies only a $0.40 above-the-line deduction and total taxable-income deductions of $24,150." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its stated computation produces tax of about $3,189 before recognizing the missing $6,000 additional senior deduction, yet it submits $626 without any corresponding deduction or credit calculation. The final value is an unsupported last-step substitution; applying the full $24,150 deduction produces $2,322.76." +us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used obsolete 10% and 15% brackets and a $9,800 deduction instead of the 2026 10% and 12% brackets and $24,150 total deduction. It also overstated taxable Social Security; the correct taxable portion is $14,050.60, producing AGI of $45,573.60." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It incorrectly assumed a post-TCJA reversion to a personal exemption, smaller standard deduction, and 15% second bracket. Current 2026 rules instead provide $24,150 in deductions, including the $6,000 additional senior deduction, and retain the 12% bracket applicable here." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its answer treats roughly the entire $58,949 of cash income as gross taxable income instead of applying the Social Security inclusion formula. Only $14,050.60 of the $27,426 benefit enters AGI, and the full $24,150 deduction then leaves $21,423 taxable." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used pre-TCJA baseline assumptions involving a personal exemption and failed to apply the current-law $6,000 additional senior deduction. The applicable deductions total $24,150, leaving taxable income of $21,423." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied post-TCJA-expiration personal-exemption and rate assumptions instead of current 2026 law. The filer receives $18,150 of standard deduction plus the $6,000 additional senior deduction, and the income above the 10% bracket is taxed at 12%, not 15%." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It correctly calculated taxable Social Security but used only a $17,450 aged standard deduction. The total deduction is $24,150 because the $18,150 standard deduction is supplemented by the $6,000 additional senior deduction." +us,scenario_085,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It correctly calculated AGI but used only an approximately $17,000 aged standard deduction. Applying the full $24,150 deduction, including the $6,000 additional senior deduction, reduces taxable income to $21,423." +us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It incorrectly treated the absence of wages and dependents as eliminating federal tax liability. Taxable retirement income, interest, and $14,050.60 of taxable Social Security produce $45,573.60 of AGI and $21,423 of taxable income after deductions." +us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that deductions and retirement credits offset the income without calculating either. The listed desired contributions do not erase liability, while taxable retirement income, interest, and taxable Social Security leave $21,423 taxable after the $24,150 deduction." +us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its unexplained $4,200 estimate is consistent with applying materially smaller deductions than the required $24,150. AGI of $45,573.60 minus the traced above-the-line and taxable-income deductions leaves $21,423, whose tax is $2,322.76." +us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It incorrectly applied a TCJA sunset regime with only $15,889 of deductions and a 15% second bracket. Current 2026 law supplies $24,150 of deductions, including the $6,000 additional senior deduction, and taxes the remaining income above the first bracket at 12%." +us,scenario_085,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It treated 85% of all Social Security benefits as taxable rather than applying the two-tier formula, which yields $14,050.60, and it used an obsolete $10,631 deduction. The applicable deduction is $24,150 and the applicable second rate is 12%, not 15%." +us,scenario_085,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It correctly found AGI near $45,574 but stopped after the $18,150 aged standard deduction. It omitted the separate $6,000 additional senior deduction, which lowers taxable income from roughly $27,424 to $21,423." +us,scenario_085,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It supplied no numeric output or explanation for the requested variable, so the required structured answer was missing." +us,scenario_085,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It included 85% of the full Social Security benefit instead of the $14,050.60 produced by the two-tier formula, then claimed substantial taxable income generated near-zero tax. Even after the full $24,150 deduction, taxable income is $21,423 and creates $2,322.76 of liability." +us,scenario_085,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It incorrectly included the full 85% maximum Social Security amount rather than the $14,050.60 yielded by the two-tier formula and omitted the $6,000 additional senior deduction. Its own recalculation produces about $4,500, yet it submits $8,587.50 without any arithmetic supporting that final jump." +us,scenario_085,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It treated exceeding the upper provisional-income threshold as making 85% of the entire Social Security benefit taxable, overstated AGI through erroneous addition, omitted the $6,000 senior deduction, and invented senior and survivor-with-dependent-children credits. There are no dependent children, and the applicable senior provision is a deduction; the correct taxable Social Security is $14,050.60 and taxable income is $21,423." us,scenario_085,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_085,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_085,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model stated the correct rule that payroll tax applies only to wages and that no wages or employment income are provided, but it submitted $1,459 instead of the $0 implied by its own reasoning. Its numeric answer is a contract-level computation/output error: it used a positive payroll tax amount without any wage base subject to Social Security or Medicare tax." -us,scenario_085,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable payroll_tax value or explanation. The required derivation is that the household has no listed wage or self-employment income, so employee Social Security, employee Medicare, Additional Medicare Tax, and mandatory employee state payroll taxes are all $0." +us,scenario_085,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly stated that the household had no wages and computed zero employee Social Security, Medicare, and state payroll taxes, but then submitted 1459 instead of carrying that zero into the value field. The submitted number is unrelated to its stated derivation." +us,scenario_085,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." us,scenario_085,snap,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_085,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_085,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model correctly limited the initial Pennsylvania tax base to $1,675 of interest but denied Tax Forgiveness by counting Social Security and retirement receipts as disqualifying eligibility income. Applying Pennsylvania's Tax Forgiveness calculation eliminates the $51.42 tentative liability." -us,scenario_085,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model invented a Pennsylvania standard deduction and treated Social Security and qualifying retirement distributions as taxable income, even though Pennsylvania's personal income tax uses specified income classes and provides no standard deduction. It then applied only an estimated partial Tax Forgiveness credit instead of the credit that reduces the liability to zero." -us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model stopped after applying 3.07% to the $1,675 of taxable interest. It omitted Pennsylvania's nonrefundable Tax Forgiveness credit, which eliminates that tentative tax." -us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,The model correctly derived $51.42 of tentative tax from taxable interest but failed to apply Pennsylvania Tax Forgiveness. That nonrefundable credit reduces the pre-refundable-credit liability to $0. -us,scenario_085,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,The model calculated the tentative Pennsylvania tax on interest and asserted that no nonrefundable credit applied. Pennsylvania Tax Forgiveness does apply and offsets the entire tentative liability; rounding $51.42 to $51 cannot substitute for that credit calculation. -us,scenario_085,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model's own tax-base reasoning produced about $51, but it abandoned that computation and submitted $500 based on unspecified additional taxable classes. It also omitted Pennsylvania Tax Forgiveness, which reduces the correctly computed tentative interest tax to $0." -us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model invented a $1,250 age-based Pennsylvania interest exclusion and therefore understated the initial taxable-interest base. It also omitted Pennsylvania Tax Forgiveness, which is the actual step that eliminates the tax on the full $1,675 of interest." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,The model stopped at the $51.42 tentative tax obtained by applying Pennsylvania's flat rate to interest. It failed to apply the nonrefundable Tax Forgiveness credit that offsets this liability completely. -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The answer merely excluded Social Security and then submitted $1,782, rather than isolating Pennsylvania-taxable income classes and applying the 3.07% rate. It also omitted Tax Forgiveness, which reduces the resulting liability to $0." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model correctly identified $1,675 of taxable interest and computed about $51 of tentative tax, but stopped there. Pennsylvania Tax Forgiveness offsets the tentative liability in full." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,The model correctly excluded the Social Security and qualifying retirement income but failed to proceed from tentative tax to tax after nonrefundable credits. Pennsylvania Tax Forgiveness reduces the $51.42 tentative interest tax to zero. -us,scenario_085,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model explicitly concluded that no nonrefundable credits applied after calculating $51.42 on interest. That conclusion omits Pennsylvania Tax Forgiveness, which offsets the entire tentative liability." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,The model calculated and rounded the tentative tax on interest while stating that no nonrefundable credit applied. Pennsylvania Tax Forgiveness is a nonrefundable credit and reduces that tentative tax to $0. -us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model ended the calculation at $1,675 times 3.07%, producing the tax before credits rather than the requested tax after nonrefundable credits. Pennsylvania Tax Forgiveness eliminates the $51.4225 tentative liability." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,The model correctly computed tentative Pennsylvania tax on the interest but omitted the Tax Forgiveness stage. The applicable nonrefundable credit offsets all $51.42. -us,scenario_085,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model applied the flat rate only to taxable interest and stopped at the resulting tentative tax. It failed to subtract Pennsylvania Tax Forgiveness, which brings the requested amount to zero." -us,scenario_085,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model improperly included the age-67 taxpayer's qualifying pension, IRA, and 401(k) distributions in Pennsylvania taxable income. It also asserted that no nonrefundable credit applied, omitting Tax Forgiveness, which eliminates the actual tentative tax on interest." -us,scenario_085,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model correctly reduced the Pennsylvania tax base to $1,675 of interest but returned the tentative tax rather than tax after nonrefundable credits. Pennsylvania Tax Forgiveness offsets that tentative tax completely." -us,scenario_085,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model explicitly stated that no nonrefundable credits applied after computing $51.42 of tentative tax. It omitted Pennsylvania Tax Forgiveness, which reduces the liability to $0." -us,scenario_085,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model denied Tax Forgiveness by treating the household's Social Security and retirement receipts as disqualifying Pennsylvania eligibility income. Under the applicable eligibility-income calculation, Tax Forgiveness offsets the full $51.42 tentative tax on interest." -us,scenario_085,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly taxed the age-67 recipient's qualifying 401(k), IRA, and private-pension distributions, inflating the Pennsylvania base from taxable interest alone to $31,523. It also omitted Pennsylvania Tax Forgiveness, which eliminates the actual tentative liability." -us,scenario_086,child1_chip_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated Medicaid ineligibility as sufficient for CHIP eligibility and omitted Georgia’s independent CHIP income test. The 12-year-old meets the age condition, but household income exceeds the applicable CHIP threshold, yielding no eligibility." -us,scenario_086,child1_medicaid_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model correctly identified that pure Georgia child Medicaid is unavailable above the child Medicaid income limit, then reversed the answer by folding PeachCare/CHIP into the Medicaid variable. PolicyEngine keeps child1_medicaid_eligible separate from child1_chip_eligible, so the higher PeachCare/CHIP pathway does not make the Medicaid category valid." -us,scenario_086,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model used Georgia's PeachCare/CHIP threshold as the Medicaid threshold and then asserted an employer-sponsored-insurance premium reduction that does not bring the child into a Medicaid category. The reference trace has MAGI at 3.44 times FPL and medicaid_category as NONE, so the child fails Georgia Medicaid regardless of any separate CHIP analysis." -us,scenario_086,child1_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model inflated or confused the household-size framing by referring to a household of 3 with an absent spouse-equivalent, then used the higher PeachCare/CHIP range as if it were Medicaid. The actual Medicaid determination uses the child's tax-unit dependent status and MAGI of 3.44 times FPL, which leaves the child in no Medicaid eligibility category." -us,scenario_086,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated age under 19 as carrying a broad high-income categorical Medicaid threshold and did not apply Georgia's income limit for a 12-year-old child. At 3.44 times FPL, the child exceeds the relevant Medicaid MAGI thresholds and qualifies through no Medicaid pathway." -us,scenario_086,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable value or explanation for child1_medicaid_eligible. The correct computation assigns medicaid_category NONE and is_medicaid_eligible False because the 12-year-old dependent's household MAGI is 3.44 times FPL, above Georgia's Medicaid child limits." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model excluded the $10,900.40 taxable portion of the Social Security dependent benefits and improperly deducted $275 of auto-loan interest. It also submitted $3,305 despite its own calculation ending at $1,925; the required taxable income is $48,412.90 and the tax after the $2,200 CTC is $3,255.55." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model used single filing status and a $14,600 standard deduction instead of head-of-household status and the $24,150 deduction. It also deducted health and employer-sponsored insurance premiums from AGI and used a $2,000 CTC instead of $2,200, then submitted a figure inconsistent with its own after-credit calculations." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model treated the dependent Social Security benefits as entirely outside the head’s income, omitting the $10,900.40 taxable portion. It also used estimated deduction and bracket values and a $2,000 CTC rather than the applicable $24,150 standard deduction, exact 2026 brackets, and $2,200 credit." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded $10,900.40 of taxable Social Security benefits and used estimated deduction, bracket, and $2,000 CTC values. It then submitted $4,673, its stated tentative tax before credits, even though the requested output is after nonrefundable credits." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model excluded the $10,900.40 taxable Social Security amount and therefore understated AGI and taxable income. It compounded that error with estimated brackets and a $2,000 CTC instead of the applicable $2,200 credit." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly reverted to pre-TCJA rules, introducing personal exemptions, a small standard deduction, 15% brackets, and a $1,000 CTC. The applicable computation instead uses a $24,150 head-of-household standard deduction, current 2026 rates, no personal exemptions, and a $2,200 CTC, while including $10,900.40 of taxable Social Security." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model excluded the $10,900.40 taxable portion of Social Security benefits, reducing taxable income far below $48,412.90. It also used estimated thresholds and a $2,000 CTC, and its submitted $3,892 contradicts its own stated $2,271 after-credit calculation." -us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a pre-TCJA-style $11,000 standard deduction, personal exemptions, 15% bracket, and $1,000 CTC rather than the applicable 2026 rules. It also deducted $1,560 of health premiums from wages and omitted $10,900.40 of taxable Social Security." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the $19,648 employer-sponsored insurance premium from the already stated gross wages and applied expired-rule personal exemptions. Gross income instead begins with the full $62,000 of wages plus $10,900.40 of taxable Social Security, followed by the $337.50 educator deduction and $24,150 head-of-household standard deduction." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used all $12,824 of Social Security benefits rather than the taxable amount of $10,900.40 and used a $15,300 deduction instead of the $24,150 head-of-household standard deduction. It also used a $2,000 CTC instead of $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a post-sunset regime with personal exemptions and a $1,000 CTC. The computation uses no personal exemptions, a $24,150 head-of-household standard deduction, the applicable 2026 rate schedule, and a $2,200 nonrefundable CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly removed $19,648 of employer-sponsored insurance premiums from the reported wages and introduced personal exemptions. Those errors reduced taxable income from the required $48,412.90 to $27,852, and it additionally used a $1,000 rather than $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model supplied no derivation beyond asserting a standard deduction and $2,000 CTC. The correct steps yield $48,412.90 of taxable income, $5,455.55 of pre-credit tax, and $3,255.55 after the applicable $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model excluded the $10,900.40 taxable Social Security amount and used an estimated $22,500 standard deduction instead of $24,150. It also reduced tax by only $2,000 rather than the applicable $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model incorrectly asserted that nonrefundable credits fully offset the liability. Tax on $48,412.90 is $5,455.55, and the $2,200 CTC leaves $3,255.55 rather than zero." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated listed medical costs as itemized deductions without establishing deductible expenses above the applicable floor and failed to apply the $2,200 child tax credit in its stated reasoning. The standard-deduction computation instead produces taxable income of $48,412.90 and tax after the CTC of $3,255.55." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model excluded $10,900.40 of taxable Social Security benefits and used an estimated $22,500 standard deduction. It also treated only $384 of the $2,200 CTC as nonrefundable, whereas the full $2,200 is usable against the $5,455.55 tentative tax." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model included only $6,412, one-half of the Social Security benefits, rather than the computed taxable amount of $10,900.40. That understated AGI and taxable income; the correct taxable income is $48,412.90 before applying rates and the $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"The model named the correct categories but miscomputed their combined result by $13.45. Applying the traced amounts produces $72,562.90 of AGI, $48,412.90 of taxable income, $5,455.55 of tax before credits, and $3,255.55 after the $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied unspecified mortgage interest despite the prompt listing only a mortgage balance and requiring unlisted amounts to be zero. It also failed to show the taxable Social Security inclusion and the exact standard-deduction computation that yields $48,412.90 of taxable income." -us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-expiration regime with a $12,389 standard deduction, personal exemptions, 15% brackets, and a $1,000 CTC. The applicable rules use a $24,150 head-of-household standard deduction, no personal exemptions, current 2026 rates, and a $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly substituted pre-TCJA-style personal exemptions, a $12,538 standard deduction, 15% brackets, and a $1,000 CTC. Using the applicable $24,150 standard deduction, no exemptions, current rates, and $2,200 CTC produces $3,255.55." -us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. -us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $275 of auto-loan interest from AGI. Removing that unauthorized deduction raises taxable income from $48,137.40 to $48,412.90 and raises the after-credit tax from $3,222.49 to $3,255.55." -us,scenario_086,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model used single filing status, a $15,000 standard deduction, and single-filer brackets instead of head-of-household treatment. It also omitted $10,900.40 of taxable Social Security benefits and used a $2,000 rather than $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model treated the head and child as married filing jointly, even though a child cannot be the head’s spouse and the correct filing status is head of household. That produced the wrong standard deduction and brackets; it also used a $300 educator deduction and $2,000 CTC instead of $337.50 and $2,200." -us,scenario_086,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model treated the $1,700 ACTC cap as an automatic refundable payment even though it calculated roughly $4,300 of tax before the $2,200 CTC. That liability absorbs the entire CTC nonrefundably, leaving no refundable CTC and total refundable credits of $0." -us,scenario_086,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly awarded the maximum ACTC after stating that the household's tax liability was sufficient to use the full $2,000 CTC. A credit fully used against liability has no remaining refundable portion, so the ACTC is $0." -us,scenario_086,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly concluded that the full CTC is absorbed nonrefundably, then contradicted that computation by inventing additional deductions and assigning a $1,700 ACTC. The prompt sets unlisted expenses to zero, and the stated tax liability leaves no unused CTC to refund." -us,scenario_086,federal_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated having a qualifying child and earnings as sufficient for a positive ACTC without first applying the CTC against federal tax liability. The household's liability uses the available CTC nonrefundably, leaving refundable CTC of $0." -us,scenario_086,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model fabricated a $384 residual refundable CTC after applying an unspecified amount nonrefundably. The household has enough pre-credit tax to absorb the entire available CTC, so no residual reaches the refundable ACTC calculation." -us,scenario_086,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model treated the $1,700 refundable cap as an entitlement for every qualifying child. The cap only limits an otherwise refundable remainder; here the full CTC is used nonrefundably against tax liability, so the refundable remainder is $0." -us,scenario_086,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for federal_refundable_credits, violating the required structured-output contract." +us,scenario_085,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly computed approximately $51.42 of tax on interest but incorrectly included Social Security and qualifying retirement income when denying PA Tax Forgiveness. Schedule SP eligibility income remains within the 100% forgiveness threshold, so the entire preliminary tax is eliminated." +us,scenario_085,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated Social Security and qualifying retirement distributions as Pennsylvania taxable income and invented a Pennsylvania standard deduction, neither of which belongs in this calculation. It also failed to apply the 100% Schedule SP Tax Forgiveness that reduces the actual preliminary interest tax to zero." +us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model stopped after applying 3.07% to the $1,675 of taxable interest. It omitted the 100% Schedule SP Tax Forgiveness credit, which eliminates that liability." +us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly isolated taxable interest and computed $51.42 of preliminary PA tax, but it never applied Schedule SP. The household qualifies for 100% Tax Forgiveness, reducing the tax to $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,The model incorrectly asserted that no nonrefundable Pennsylvania credit applied. Schedule SP provides 100% Tax Forgiveness at this household's eligibility income and offsets the full tax on interest. +us,scenario_085,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's own calculation produced about $51 from taxable interest, then replaced it with $500 through unsupported rounding for unspecified additional income classes. It also omitted the 100% Schedule SP Tax Forgiveness that reduces the supported $51.42 preliminary liability to zero." +us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model invented a $1,250 age-based exclusion from Pennsylvania taxable interest and therefore computed the wrong preliminary tax base. The full $1,675 interest amount produces about $51.42 before the 100% Schedule SP Tax Forgiveness eliminates it." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,The model stopped at the $51.42 tax on interest and omitted Pennsylvania Schedule SP. The applicable 100% Tax Forgiveness offsets the entire preliminary liability. +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $1,782 implies a broad income base beyond Pennsylvania-taxable interest, despite the explanation recognizing only the Social Security exclusion. Qualifying retirement distributions are also excluded, and the remaining tax on $1,675 of interest is fully eliminated by Schedule SP Tax Forgiveness." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model correctly derived roughly $51 of preliminary tax from the taxable interest but stopped before credits. The household receives 100% Schedule SP Tax Forgiveness, reducing that amount to $0." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,The model correctly computed $51.42 before credits but failed to apply Schedule SP Tax Forgiveness. The 100% forgiveness percentage offsets the entire liability. +us,scenario_085,state_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model correctly identified the $1,675 taxable-interest base but incorrectly concluded that no applicable credit followed. Schedule SP grants 100% Tax Forgiveness and reduces the $51.42 preliminary tax to zero." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,The model explicitly applied no nonrefundable credits after calculating tax on interest. It omitted the 100% Schedule SP Tax Forgiveness that eliminates the preliminary liability. +us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model stopped at the raw 3.07% tax on $1,675 of interest. Pennsylvania's 100% Schedule SP Tax Forgiveness offsets the resulting $51.4225 completely." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,The model calculated the pre-forgiveness tax on interest but did not apply Schedule SP. The household's 100% Tax Forgiveness reduces $51.42 to $0. +us,scenario_085,state_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,The model's $51 answer reflects only the flat tax on taxable interest. It omitted the 100% Schedule SP Tax Forgiveness that offsets this entire amount. +us,scenario_085,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly taxed the qualifying pension, IRA, and 401(k) distributions, which Pennsylvania excludes here. Only $1,675 of interest generates preliminary tax, and Schedule SP Tax Forgiveness then eliminates that tax." +us,scenario_085,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,The model correctly limited the Pennsylvania base to interest but stopped after computing the flat tax. It omitted the 100% Schedule SP Tax Forgiveness that reduces the result to zero. +us,scenario_085,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,The model incorrectly included qualifying retirement distributions in Pennsylvania taxable income and in its denial of Tax Forgiveness. Those distributions are excluded here; the remaining interest tax is fully offset by 100% Schedule SP Tax Forgiveness. +us,scenario_085,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,credit_phaseout,False,"The model correctly computed $51.42 of preliminary tax but incorrectly stated that no nonrefundable credit applied. The household qualifies for 100% Schedule SP Tax Forgiveness, which offsets the full amount." +us,scenario_085,state_income_tax_before_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model incorrectly treated excluded Social Security and qualifying retirement benefits as making Schedule SP eligibility income too high. The applicable eligibility income remains within the full-forgiveness threshold, so the $51.42 interest tax is entirely forgiven." +us,scenario_085,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model wrongly included the pension, IRA, and 401(k) distributions in Pennsylvania taxable income. Only the $1,675 of interest is taxable, and the resulting $51.42 preliminary tax is fully eliminated by Schedule SP Tax Forgiveness." +us,scenario_086,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated Medicaid ineligibility as sufficient for CHIP eligibility and never applied Georgia’s CHIP income limit. Although the 12-year-old satisfies the age criterion, the household fails the CHIP income criterion, so the child is not eligible." +us,scenario_086,child1_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly found that income exceeds Georgia’s child Medicaid limit, then reversed its result by treating PeachCare/CHIP as part of PolicyEngine’s Medicaid output. Medicaid and CHIP are separate benchmark variables, and child1 qualifies for no Medicaid category at 3.44 times FPL." +us,scenario_086,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the head’s employer-sponsored insurance premiums from Medicaid MAGI and then asserted that $74,824 was within a roughly $50,487 limit. Those premiums do not reduce MAGI on the stated facts, and 3.44 times FPL exceeds Georgia’s child Medicaid thresholds." +us,scenario_086,child1_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model invented a three-person household despite the prompt listing only the head and child, then conflated the 247% FPL PeachCare/CHIP ceiling with Medicaid eligibility. For the actual two-person household, child1 is at 3.44 times FPL and has no Medicaid eligibility category." +us,scenario_086,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model relied on age-based categorical eligibility without applying Georgia’s income threshold for a 12-year-old child. Child1’s MAGI income level is 3.44 times FPL, above the applicable child Medicaid limits, leaving the Medicaid category as NONE." +us,scenario_086,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_medicaid_eligible, violating the required output contract." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model excluded the $10,900.40 taxable portion of the Social Security dependent benefits and incorrectly deducted $275 of auto-loan interest. It also submitted $3,305 despite its own stated computation yielding $1,925." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model used single filing status instead of head of household, deducted employer and other health premiums from AGI, and used the wrong standard deduction. It also failed to apply the 2026 $2,200 child tax credit and submitted a value inconsistent with each calculation in its explanation." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model excluded $10,900.40 of taxable Social Security benefits, understated the head-of-household standard deduction, and used a $2,000 child tax credit instead of $2,200. Those errors reduced its taxable income and final liability." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded $10,900.40 of taxable Social Security benefits and used estimated deduction and bracket values instead of the 2026 parameters. It then submitted the asserted pre-credit tax of $4,673 without subtracting the $2,000 credit described in its own reasoning." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated all Social Security dependent benefits as nontaxable, although $10,900.40 enters gross income, and it used estimated deduction, bracket, and credit values. The correct taxable-income base is $48,412.90, not roughly $37,162." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly assumed the 2026 reversion of pre-TCJA rules, introducing personal exemptions, a small standard deduction, 15% brackets, and a $1,000 child tax credit. The applicable 2026 rules instead provide a $24,150 head-of-household standard deduction and a $2,200 credit." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model excluded $10,900.40 of taxable Social Security benefits and used estimated deduction, bracket, and credit parameters. It also submitted $3,892 even though its stated tax-minus-credit calculation equals $2,271." +us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a pre-TCJA-style $11,000 standard deduction, personal exemptions, 15% bracket, and $1,000 child tax credit. It also deducted $1,560 of health premiums from wages while omitting the $10,900.40 taxable Social Security amount." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the $19,648 employer-sponsored insurance premium from the stated gross wages even though the wage input is the tax-model income amount, then applied expired personal exemptions and a $1,000 credit. This understated AGI and taxable income substantially." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The answer used a $15,300 deduction rather than the $24,150 head-of-household standard deduction and treated the full $12,824 benefit as gross income rather than including $10,900.40 as taxable. It also used a $2,000 credit instead of $2,200 and omitted the traced educator adjustment." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly applied post-sunset personal exemptions and a $1,000 child tax credit. The 2026 computation uses no personal exemptions, a $24,150 head-of-household standard deduction, and a $2,200 nonrefundable child tax credit." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by $19,648 of employer-sponsored premiums and then used personal exemptions, a $15,000 standard deduction, and a $1,000 credit. The traced AGI is $72,562.90 and taxable income is $48,412.90." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The unexplained estimate does not reproduce the traced sequence of $72,900.40 gross income, $337.50 educator deduction, $24,150 standard deduction, $5,455.55 tentative tax, and $2,200 credit. Its $3,678 result leaves $422.45 too much liability after the applicable nonrefundable credit." +us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model excluded the $10,900.40 taxable Social Security amount and used estimated 2026 deduction and bracket values. It also applied a $2,000 child tax credit rather than $2,200." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated nonrefundable credits as sufficient to eliminate the entire tentative tax. The traced tentative tax is $5,455.55 and the applicable nonrefundable child tax credit is only $2,200, leaving $3,255.55." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked unspecified medical itemized deductions even though the household claims the $24,150 standard deduction, and it assumed no applicable nonrefundable credit. The computation must include $10,900.40 of taxable Social Security and subtract the $2,200 child tax credit." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model excluded taxable Social Security benefits and incorrectly treated only $384 of the child tax credit as reducing pre-refundable liability. The full $2,200 nonrefundable credit is usable against the $5,455.55 tentative tax." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model included only $6,412 of Social Security benefits, one-half of the gross benefit, rather than the traced taxable amount of $10,900.40. That understated AGI and taxable income before applying the otherwise recognized $2,200 credit." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"The model identified the correct components but did not compute them exactly. Those components yield $48,412.90 of taxable income, $5,455.55 of tentative tax, and $3,255.55 after the $2,200 credit, not $3,269." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model introduced mortgage interest despite receiving only a mortgage balance and no deductible interest amount. It failed to follow the traced standard-deduction computation and overstated tax after the child credit by $1,267.45." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied pre-TCJA personal exemptions, a $12,389 standard deduction, 15% brackets, and a $1,000 child tax credit. The applicable computation uses the $24,150 head-of-household standard deduction, current 2026 brackets, and a $2,200 credit." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied a post-sunset regime with personal exemptions, a $12,538 standard deduction, 15% brackets, and a $1,000 credit. Those are not the 2026 parameters used for this household; the applicable deduction is $24,150 and the credit is $2,200." +us,scenario_086,federal_income_tax_before_refundable_credits,inkling,llm_error,other,False,"The model followed the correct derivation but rounded the educator deduction, taxable Social Security amount, taxable income, and tentative tax. Exact traced inputs produce $5,455.55 before credits and $3,255.55 after the $2,200 credit." +us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable output for the requested variable. +us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $275 of auto-loan interest from AGI. Removing that deduction raises taxable income from $48,137.40 to $48,412.90 and yields $3,255.55 after the child tax credit." +us,scenario_086,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model used single filing status and a $15,000 standard deduction instead of head-of-household status and the $24,150 deduction. It also excluded $10,900.40 of taxable Social Security and used a $2,000 credit instead of $2,200." +us,scenario_086,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model treated the head and child as married filing jointly, even though a child cannot be the head's spouse and the filing status is head of household. It consequently used the married-joint standard deduction and brackets and also used a $2,000 credit instead of $2,200." +us,scenario_086,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that the standard deduction and Social Security exclusion reduce taxable income to zero, but the traced taxable income is $48,412.90. The $2,200 child tax credit reduces the $5,455.55 tentative tax only to $3,255.55, not zero." +us,scenario_086,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model computed about $4,300 of tax and a $2,200 Child Tax Credit but then claimed the nonrefundable credit absorbed the tax and produced a $1,700 ACTC. The direction is reversed: the tax liability absorbs the entire CTC, leaving no unused CTC for the refundable ACTC." +us,scenario_086,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model treated the $1,700 ACTC cap as an automatic refundable amount despite acknowledging substantial federal tax liability. The full Child Tax Credit is used nonrefundably against that liability, leaving no refundable ACTC." +us,scenario_086,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model correctly established that roughly $4,300 of liability absorbs the entire $2,000 CTC and explicitly derived no ACTC, then contradicted that derivation by inventing a $1,700 refundable remainder. No listed deduction reduces liability below the CTC, and deductions would not justify selecting the maximum ACTC without recomputing the unused credit." +us,scenario_086,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model assumed that having a qualifying child and earnings automatically creates a refundable ACTC. At $62,000 of wages, the filer has enough federal income tax liability to use the CTC nonrefundably, and the earnings are above the one-child EITC range, yielding zero refundable credits." +us,scenario_086,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model invented a $384 unused portion of the Child Tax Credit without computing the tax liability against which the nonrefundable credit is applied. The liability exceeds the full CTC, so none of the credit remains for the ACTC." +us,scenario_086,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model treated the $1,700 refundable CTC cap as an entitlement for every qualifying child. The cap only limits an otherwise refundable remainder; here the entire CTC is used against federal income tax liability, so the ACTC is zero." +us,scenario_086,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, violating the required submission contract." +us,scenario_086,federal_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model applied the credit by reference to the child's separate tax liability, but the Child Tax Credit belongs to the head's tax unit and offsets the head's federal income tax liability. That liability uses the entire CTC nonrefundably; the child's dependent Social Security benefits do not make the CTC fully refundable." us,scenario_086,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_086,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_086,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model correctly computed federal employee FICA on $62,000 but then added a nonexistent Georgia employee payroll tax of $466.80. Georgia unemployment insurance is an employer-side unemployment tax, not an employee-side mandatory payroll tax included in this output." -us,scenario_086,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model used $60,440 as the FICA wage base instead of the head's full $62,000 of gross wages. The $1,560 health insurance premium input does not reduce the payroll-tax wage base in this benchmark calculation, so both Social Security and Medicare taxes were understated." -us,scenario_086,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the $19,648 employer-sponsored insurance premium from wages and applied FICA only to $42,352. PolicyEngine's payroll tax calculation applies employee Social Security and Medicare taxes to the full $62,000 of wages, so excluding the insurance premium removed too much payroll-taxable compensation." -us,scenario_086,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model treated the $19,648 employer-sponsored insurance premium as a pre-tax employee wage reduction for FICA and used $42,352 as taxable wages. The payroll tax output taxes the head's full $62,000 of wages for employee Social Security and Medicare, producing $4,743 rather than about $3,240." -us,scenario_086,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model computed Social Security correctly at $3,844 but failed to carry through the Medicare component correctly: 1.45% of $62,000 is $899, not the $135 implicit in its $3,979 total. Its stated rejection of additional Medicare tax and state employee payroll taxes is consistent with the reference, but the arithmetic omits most of regular Medicare tax." -us,scenario_086,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable payroll_tax value or explanation. The required output was the head's employee FICA on $62,000 of wages, $3,844 in Social Security plus $899 in Medicare, totaling $4,743." -us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model substituted a $24,000 head-of-household standard deduction and $4,000 dependent exemption for the applicable $15,000 and $5,000 amounts. It then submitted $2,732 despite its own recomputation yielding $1,749, adding an internal arithmetic-output inconsistency." -us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model applied Georgia's obsolete graduated 1%–6% single-filer brackets directly to $62,000 of wages. It omitted the federal-AGI starting point, Georgia's Social Security subtraction, the $15,000 head-of-household standard deduction, and the $5,000 dependent exemption." -us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used an $18,500 standard deduction and $4,000 dependent exemption instead of $15,000 and $5,000, producing the wrong taxable-income base. Its submitted $2,191 also does not follow from its stated $39,162 base and 5.19% multiplication." -us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model treated the head of household as effectively single, using a $12,000 standard deduction and $4,000 dependent exemption rather than $15,000 and $5,000. It also used the wrong rate and submitted $1,869 even though its own stated multiplication yielded $2,324." -us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model double-subtracted the entire $12,824 Social Security benefit after constructing federal AGI without its $10,900.40 taxable portion. It also used obsolete $5,400, $2,700, and $3,000 deductions and exemptions instead of the applicable $15,000 head-of-household standard deduction and $5,000 dependent exemption." -us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the taxable Social Security component from federal AGI and used a $12,000 standard deduction plus $4,000 dependent exemption instead of $15,000 and $5,000. Its final $2,765 is also disconnected from its stated $2,372 calculation." -us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model constructed the wrong AGI and applied obsolete deductions consisting of a $5,400 standard deduction and two $2,700 personal exemptions. The correct base is $61,662.50 of Georgia AGI less $15,000 and $5,000, leaving $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model began from an unsupported federal AGI of $52,952.40 instead of $72,562.90 and used a $15,800 personal exemption plus $3,000 dependent exemption. It therefore never formed the traced Georgia taxable income of $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave only a wage-based estimate and did not account for federal AGI including taxable Social Security, Georgia's Social Security subtraction, or the applicable $15,000 and $5,000 deductions. Its $2,690 reflects an unsupported shortcut rather than the $41,662.50 taxable-income derivation." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model correctly reached Georgia AGI near $61,662 but used a $12,000 standard deduction and $3,000 dependent deduction. Applying the required $15,000 standard deduction and $5,000 dependent exemption instead leaves $41,662.50 taxable." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an unsupported federal AGI of $52,952 and an unspecified deduction package to obtain $37,952 of taxable income. The trace instead yields Georgia AGI of $61,662.50 and taxable income of $41,662.50 after the $15,000 and $5,000 deductions." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The unexplained $3,047 answer does not incorporate the traced Georgia taxable-income calculation. Federal AGI of $72,562.90, the $10,900.40 Social Security subtraction, and deductions of $15,000 and $5,000 yield $41,662.50 taxable, not the higher base implied by its answer." -us,scenario_086,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model correctly reconstructed Georgia AGI near $61,662 but allowed only a $10,600 head-of-household deduction and omitted the $5,000 dependent exemption. Those errors inflated taxable income from $41,662.50 to $51,062." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model supplied no deduction amounts or rate computation, and its $1,639 implies an excessive reduction of the Georgia tax base. The specified Social Security subtraction, $15,000 standard deduction, and $5,000 dependent exemption leave $41,662.50 taxable and produce $2,078.96." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model imported federal-style standard, educator, and medical deductions into the Georgia calculation and reduced the liability to $498. The traced state calculation instead starts from Georgia AGI of $61,662.50 and subtracts only the applicable $15,000 standard deduction and $5,000 dependent exemption before applying the state schedule." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model omitted the taxable Social Security component from federal AGI and used a $12,000 standard deduction with only $4,500 of exemptions. The correct state base is $61,662.50 less $15,000 and $5,000, or $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $1,357.80 implies that it over-reduced income when applying the head-of-household deduction and dependent exemption. The applicable deductions total $20,000 against Georgia AGI of $61,662.50, leaving $41,662.50 taxable." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used $12,000 and $4,000 deductions instead of the applicable $15,000 standard deduction and $5,000 dependent exemption, overstating taxable income by $4,000. It also applied the wrong 5.09% rate rather than the schedule that produces $2,078.96." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $1,903 does not follow the applicable head-of-household deduction and dependent exemption. Georgia AGI of $61,662.50 less $15,000 and $5,000 leaves $41,662.50, which the state schedule taxes at $2,078.96." -us,scenario_086,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model did not identify its taxable-income base, deduction amounts, or rate schedule. Its $2,184 fails to follow the traced $61,662.50 Georgia AGI less the $15,000 standard deduction and $5,000 dependent exemption." -us,scenario_086,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model combined the standard deduction and dependent exemption into an incorrect $18,000 reduction. The applicable reductions total $20,000—$15,000 for head of household and $5,000 for the dependent—leaving $41,662.50 rather than $43,662." -us,scenario_086,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted the $338 educator deduction after starting from federal AGI, then applied obsolete $5,400 standard-deduction and $5,400 exemption amounts. The correct Georgia AGI is $61,662.50 and the applicable deductions are $15,000 and $5,000." -us,scenario_086,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model reached Georgia AGI near $61,662 but used a $12,000 standard deduction and two $2,700 personal exemptions instead of a $15,000 standard deduction and $5,000 dependent exemption. It also applied an incorrect 4.99% rate." -us,scenario_086,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model misstated Georgia AGI as $61,387 and used an $18,500 standard deduction plus a $3,000 dependent exemption. The trace uses $61,662.50 less $15,000 and $5,000, producing $41,662.50 of taxable income." -us,scenario_086,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model calculated from wages alone, used a roughly $5,000 standard deduction and only a $2,700 personal exemption, and omitted the dependent exemption. The applicable calculation uses Georgia AGI of $61,662.50, a $15,000 head-of-household standard deduction, and a $5,000 dependent exemption." -us,scenario_086,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $12,824 Social Security benefit as Georgia income instead of starting with federal AGI and subtracting its $10,900.40 taxable portion for Georgia. It also used incorrect $12,000, $2,700, and $3,000 deduction amounts and the wrong 5.12% rate, rather than deriving $41,662.50 of taxable income." -us,scenario_086,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model fabricated a Georgia refundable $260 per-child credit for a qualifying dependent under age 17. Georgia does not provide that refundable state child credit in the benchmark computation, so the correct Georgia refundable-credit amount is $0 and the national state-refundable-credit aggregate remains $0." -us,scenario_088,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $15,000 base standard deduction plus only a $1,950 blind addition, leaving $7,085 of taxable income taxed at 10%. It missed that the 2026 single-filer standard deduction threshold used by PolicyEngine is high enough to wipe out the $23,359.76 AGI, so no regular income tax remains before refundable credits." -us,scenario_088,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable value or explanation for federal_income_tax_before_refundable_credits. The required output was a numeric amount with a supporting explanation, and the correct derivation yields $0 because AGI is below the applicable standard deduction threshold." -us,scenario_088,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model submitted a negative federal income tax before refundable credits, which is not a valid result for tax before refundable credits after nonrefundable credits. It mixed deduction concepts and rounding into an amount below zero instead of capping the regular tax computation at $0 after the standard deduction eliminates taxable income." -us,scenario_088,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the state tax refund and reached AGI of about $23,360, but then applied an understated 2026 standard deduction of about $16,550. That left artificial taxable income of $6,810; under the PolicyEngine 2026 standard deduction calculation, AGI falls below the deduction threshold and regular tax before refundable credits is $0." -us,scenario_088,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model calculated AGI of approximately $24,035 but applied the childless EITC phaseout only to $18,577 of earned income. Using the greater of AGI and earned income places the filer beyond the no-child phaseout endpoint, yielding $0 rather than $632." -us,scenario_088,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for federal_refundable_credits, violating the required output contract." -us,scenario_088,federal_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model phased out the childless EITC using only $18,577 of earned income and ignored the higher approximately $24,035 AGI. The phaseout uses the greater amount, which eliminates the credit entirely." -us,scenario_088,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated gross earned income below the nominal limit as sufficient for a positive childless EITC. It failed to apply the phaseout using the higher approximately $24,035 AGI, which reduces the credit to $0." -us,scenario_088,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model compared only $18,577 of earned income with the childless EITC limit and assigned an amount near the maximum credit. The EITC phaseout instead uses the greater of earned income and approximately $24,035 AGI, placing this filer beyond the phaseout endpoint and producing $0." -us,scenario_088,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated disability as sufficient to enter the aged/blind/disabled Medicaid pathway in Texas. PolicyEngine did not assign any Medicaid category to the person, and the stated income does not create ABD eligibility absent an SSI-related qualifying pathway; the correct output is not eligible." -us,scenario_088,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly calculated SSI-related countable income above the SSI Federal Benefit Rate, then overrode that result by assuming PolicyEngine applies a broader disabled/blind Medicaid pathway. PolicyEngine's trace assigns medicaid_category = NONE and SSI received = 0, so blind and disabled status alone does not qualify this Texas adult for Medicaid." -us,scenario_088,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model assumed disabled and blind status qualifies the head through an SSI-related aged/blind/disabled pathway and also asserted that $31,776 in assets fits typical SSI-related limits. PolicyEngine finds no qualifying Medicaid category, no SSI receipt, and no applicable Texas pathway, so the disability facts do not overcome the failed eligibility tests." -us,scenario_088,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model collapsed Medicaid eligibility to disabled plus low income. PolicyEngine requires a qualifying Medicaid category, and this Texas adult has medicaid_category = NONE with MAGI income at 1.46 FPL, so the shortcut produces the wrong yes answer." -us,scenario_088,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model invented an eligibility pathway from disability status and housing assistance. PolicyEngine's Medicaid trace contains no Texas pathway for Section 8 or housing assistance recipients here, and the person qualifies through none of the MAGI or categorical Medicaid categories." -us,scenario_088,head_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the disability flag as sufficient for Medicare eligibility regardless of age. Disability alone does not establish the required SSDI entitlement and waiting-period pathway, and the 56-year-old head therefore remains ineligible." -us,scenario_088,head_medicare_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model correctly named the 24-month SSDI pathway but then inferred SSDI receipt and completion of that period solely from the disability flag. Those unlisted facts are false under the prompt, so the head does not qualify for Medicare before age 65." -us,scenario_088,head_medicare_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that disability qualifies a person for Medicare regardless of age. The under-65 pathway requires qualifying SSDI entitlement and its waiting period or another specified condition, none of which is present." -us,scenario_088,head_medicare_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model invented a PolicyEngine rule that makes the disability boolean independently sufficient for Medicare eligibility. PolicyEngine requires an actual under-65 qualifying pathway, and no SSDI entitlement period, end-stage renal disease, or other qualifying condition was supplied." -us,scenario_088,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly removed the $9,600 of tips from Social Security wages and then added a separate tip FICA calculation, even though the prompt states tips are already included in gross wages and employee payroll tax is computed on total employee wages. It also invented an employee tip-credit reduction that does not reduce the worker's FICA liability." -us,scenario_088,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model zeroed out employee FICA despite $18,577 of annual wage and salary income. Employee-side Social Security and Medicare taxes apply to those wages in Texas, while the absence of mandatory state payroll tax only makes the state payroll component zero." -us,scenario_088,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model applied the right combined employee FICA rates to wages but rounded the result to a rough estimate instead of carrying the exact Social Security and Medicare calculations. PolicyEngine computes $18,577 times 6.2% for Social Security plus $18,577 times 1.45% for Medicare, yielding $1,421.14 rather than the submitted $1,270." -us,scenario_088,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable payroll_tax output. The required employee payroll tax calculation is Social Security tax plus Medicare tax on the household's wage income, which yields $1,421.14." -us,scenario_088,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model applied the disabled-household net-income pathway but omitted the SNAP resource test. The $31,776 bank balance exceeds the applicable resource limit, making the household ineligible before its earned-income, standard, medical, and shelter deductions can generate a benefit." -us,scenario_088,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model ignored the disqualifying $31,776 bank balance and proceeded directly to net-income deductions. It also incorrectly capped the excess-shelter deduction despite the disabled member, but the resource-limit failure already reduces SNAP to zero." -us,scenario_088,snap,glm-5.2,llm_error,asset_resource,False,"The model treated large shelter expenses and disability as sufficient to reduce net income to zero and award the maximum allotment, without applying the resource test. The $31,776 bank balance exceeds the applicable SNAP resource limit, so no allotment is payable." -us,scenario_088,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model correctly recognized the disabled-household net-income pathway and uncapped shelter deduction but omitted the separate resource-eligibility step. The $31,776 in bank assets disqualifies the household regardless of the calculated shelter deduction or maximum allotment." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $3,467.43 overtime deduction, leaving taxable income at about $64,432 instead of $60,964.27. Its submitted $8,225 also contradicts its own correctly separated regular-income-tax calculation of roughly $7,236." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented $57,893 of head wages despite the instruction that unlisted income is zero, and then applied unspecified health-premium and credit adjustments. It also omitted the qualified business income and overtime deductions that reduce taxable income to $60,964.27." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model understated the standard deduction, approximated the QBI deduction instead of using $9,568.44, and omitted the $3,467.43 overtime deduction. Its $9,089 submission also does not follow from its own estimated $7,377 regular tax computation." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model added approximately $4,286 of self-employment tax to regular federal income tax even though this output excludes self-employment tax. It also omitted both the $9,568.44 QBI deduction and the $3,467.43 overtime deduction from taxable income." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's stated deductions and brackets produce roughly $7,150, but it submitted $12,850 after an unsupported upward adjustment for business income. Partnership and self-employment income were already included in AGI and cannot be added again through an unspecified higher effective rate." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model deducted the head's $5,403 desired traditional 401(k) contribution even though the head had no wages, then omitted the $3,467.43 overtime deduction. Those errors largely offset, but its estimated $30,800 standard deduction remained below the applicable $32,200 amount, producing $6,761 instead of $6,819.71." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model treated the $19,650 partnership/S-corp income as self-employment earnings, inflating the self-employment-tax deduction and failing to calculate the $9,568.44 QBI deduction. It also omitted the $3,467.43 overtime deduction and used an understated standard deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced wages to $57,735 without support, used a $29,200 standard deduction, calculated QBI as 20% of unreduced business receipts, and omitted the overtime deduction. The correct deductions are $32,200 standard, $9,568.44 QBI, and $3,467.43 overtime." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA individual provisions expired for 2026 and substituted personal exemptions, a $16,000 standard deduction, and a 15% bracket. It also subtracted the listed employer-sponsored insurance premium from wages even though the trace's wage reduction is the $6,946 traditional 401(k) contribution." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies a rough combined-income calculation that did not derive taxable income through the $32,200 standard deduction, $9,568.44 QBI deduction, and $3,467.43 overtime deduction. Those deductions reduce taxable income to $60,964.27 before the 2026 brackets are applied." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a presumed TCJA sunset, replacing current 2026 law with a small standard deduction, personal exemptions, and pre-TCJA brackets. It also improperly subtracted $7,789 of employer-sponsored insurance premiums from wages." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used obsolete personal exemptions, a $15,300 standard deduction, and a 15% bracket rather than the applicable 2026 married-filing-jointly rules. It also reduced wages by the employer-sponsored insurance premium and omitted the QBI and overtime deductions." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $7,789 of employer-sponsored insurance premiums from wages, lowering AGI to $98,410.90 instead of $106,200.13. It also omitted the $3,467.43 overtime deduction and used a $30,800 rather than $32,200 standard deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model correctly derived AGI and the QBI deduction but used only a $30,600 standard deduction and omitted the $3,467.43 overtime deduction. The applicable deductions reduce taxable income to $60,964.27, not $66,031.53." -us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model ignored the prompt's explicit assumptions that all listed people form the tax-benefit unit, filing occurs when required, and unlisted inputs are zero. The supplied facts determine $106,200.13 of AGI, $60,964.27 of taxable income, and positive federal income tax." -us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model supplied only a generic estimate and did not carry the stated inputs through the exact deduction and bracket calculation. Tax on the trace-derived $60,964.27 of taxable income is $6,819.71, not the rounded $6,828 estimate." -us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model referred to pretax health deductions and estimated taxable income at $62,996 instead of deriving $60,964.27. The listed employer-sponsored insurance premiums are not an additional wage deduction in this trace, while the $9,568.44 QBI deduction must be included alongside the overtime and standard deductions." -us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model named the retirement, half-SE-tax, standard, and QBI deductions but omitted the separate $3,467.43 overtime deduction from its explanation and did not reach $60,964.27 of taxable income. Its $7,530 answer reflects an overstated taxable-income base." -us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used an imprecise income base of about $120,000 and did not apply the above-the-line, QBI, and overtime deductions shown by the household facts. The correct sequence produces $106,200.13 of AGI and $60,964.27 of taxable income." -us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly imposed a TCJA sunset and used personal exemptions, a $16,637 standard deduction, and a 15% bracket. Under the applicable 2026 rules, the standard deduction is $32,200, personal exemptions do not replace it, and the relevant rates are 10% and 12%." -us,scenario_089,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted the $3,467 overtime premium by adding it to gross wages even though gross wages already include overtime, and it deducted the head's unfunded $5,403 401(k) contribution. It also omitted the $9,568.44 QBI deduction and the statutory overtime deduction from taxable income." -us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model added the $3,467 overtime premium to wages even though the prompt states that gross wages already include overtime, then deducted both spouses' desired 401(k) amounts despite the head having no wages. It also omitted both the $9,568.44 QBI deduction and the $3,467.43 overtime deduction from taxable income." -us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model derived taxable income essentially correctly, with only a small rounding difference, but applied the wrong 2026 bracket parameters. Applying the engine's 2026 married-filing-jointly rates to $60,964.27 yields $6,819.71 rather than $6,827.74." -us,scenario_089,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated both spouses' desired 401(k) contributions as deductible and calculated half of self-employment tax on both self-employment and partnership income, understating AGI. It also used the unreduced $49,985 business-income total for QBI, omitted the overtime deduction, and understated the standard deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the head's $5,403 desired 401(k) contribution despite the head having no wages, omitted the $9,568.44 QBI deduction, and omitted the $3,467.43 overtime deduction. It also used an incorrect bracket breakpoint and base-tax amount rather than applying the 2026 schedule to $60,964.27." -us,scenario_089,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model invented wage income for the head and added fabricated North Carolina employee payroll taxes, even though unlisted income is zero and North Carolina imposes no mandatory employee payroll tax included here. It also initially conflated self-employment and partnership income with W-2 wages; only the spouse's $65,881 wages generate the $5,039.91 employee FICA liability." -us,scenario_089,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model identified the correct $65,881 FICA wage base but made an arithmetic error in Medicare tax and then submitted $5,085.44, a value unsupported by any calculation in its explanation. The trace calculation is $4,084.63 of Social Security plus $955.28 of Medicare, totaling $5,039.91." -us,scenario_089,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly deducted $1,200 of separately listed health-insurance premiums from FICA wages. PolicyEngine applies employee Social Security and Medicare taxes to the full $65,881 wage input, producing $5,039.91." -us,scenario_089,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the $7,789 employer-sponsored insurance premium as an employee pre-tax payroll deduction and subtracted it from wages. That input does not reduce the spouse's $65,881 FICA wage base, so employee FICA totals $5,039.91." -us,scenario_089,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model added the $3,467 FLSA overtime premium to the $65,881 annual gross-wage total, despite the prompt specifying that gross wages already include overtime pay. FICA applies to $65,881, not $69,348, yielding $5,039.91." -us,scenario_089,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly assumed the $7,789 employer-sponsored insurance premium was a pre-tax employee contribution deductible from FICA wages. The spouse's full $65,881 wage input remains subject to Social Security and Medicare taxes, totaling $5,039.91." -us,scenario_089,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages by the $7,789 employer-sponsored insurance premium. PolicyEngine taxes the full $65,881 wage amount at the employee Social Security and Medicare rates, producing $5,039.91." -us,scenario_089,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model misclassified the $7,789 employer-sponsored insurance premium as a pre-tax employee payroll deduction and subtracted it from FICA wages. The correct FICA base is the full $65,881 of wages, resulting in $5,039.91." -us,scenario_089,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model correctly stated that only the spouse's $65,881 wages generate employee Social Security and Medicare tax, but its $8,107 output does not follow from that basis. Applying the combined 7.65% employee FICA rate gives $5,039.91, with no Additional Medicare Tax." -us,scenario_089,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model taxed wages for both spouses even though the head has no listed wage income and unlisted numeric inputs are zero. Only the spouse's $65,881 is subject to employee FICA, producing $5,039.91." -us,scenario_089,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model incorrectly excluded the $7,789 employer-sponsored insurance premium from the spouse's FICA wage base. The full $65,881 wage amount generates $4,084.63 of Social Security tax and $955.28 of Medicare tax, totaling $5,039.91." -us,scenario_089,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model's $8,120 output is inconsistent with employee FICA on the spouse's wages and overtime. The $65,881 annual gross-wage input already includes overtime, and applying 6.2% Social Security plus 1.45% Medicare yields $5,039.91." -us,scenario_089,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model double-counted the $3,467 FLSA overtime premium by adding it to annual gross wages, although the prompt states gross wages include all overtime pay. Employee FICA therefore applies to $65,881 rather than $69,348, yielding $5,039.91." -us,scenario_089,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model correctly calculated employee FICA on the spouse's wages but then added half of the spouse's self-employment tax to payroll tax. Self-employment tax is a separate requested output and is excluded in full from employee-side payroll tax, leaving $5,039.91." -us,scenario_089,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model identified the correct self-employment tax pathway but submitted a number inconsistent with its own formula. Its written calculation applies the 92.35% adjustment and 15.3% rate to the spouse's $30,335 of self-employment income, while the submitted $4,275.23 reflects an arithmetic or transcription error rather than the computed tax." -us,scenario_089,self_employment_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model used the correct income item and correctly excluded the partnership/S-corp amount, but it miscomputed the 92.35% net-earnings base for $30,335. That inflated the adjusted base and produced $4,287.72 instead of the self-employment tax from the engine's net-earnings computation." -us,scenario_089,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model incorrectly added the $19,650 partnership or S-corp income to the spouse's stated $30,335 self-employment income. It then applied the 92.35% adjustment and 15.3% tax rate to that enlarged base, so its answer is driven by taxing income PolicyEngine did not include in self_employment_tax." -us,scenario_089,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model stated the correct high-level formula but computed the product incorrectly. Applying the self-employment tax calculation to $30,335 after the 92.35% net-earnings adjustment yields the reference amount, while $4,357 is consistent with an arithmetic error in the adjusted base or rate application." -us,scenario_089,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model correctly excluded the partnership/S-corp income but failed to apply the 92.35% net-earnings adjustment before applying self-employment tax. Its $5,114 answer is consistent with taxing too large a base or using a blended shortcut rather than the statutory adjusted self-employment earnings base." -us,scenario_089,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model described the standard 92.35% adjustment and 15.3% rate but submitted a value far below that calculation. Its answer reflects an arithmetic error or an improper reduction to the self-employment tax base after excluding Additional Medicare Tax, which is not part of this output." -us,scenario_089,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model described taxing self-employment and partnership income after deductions, which is not the base PolicyEngine used for this output. The correct computation uses the spouse's stated self-employment income with the 92.35% adjustment; partnership/S-corp income and unspecified deductions are not added into the self-employment tax base." -us,scenario_089,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable self_employment_tax value or supporting explanation. This is a missing-output contract failure rather than a substantive tax calculation. -us,scenario_089,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model incorrectly combined $30,335 of self-employment income with $19,650 of partnership or S-corp income and treated the full $49,985 as self-employment earnings. That overstated both the Social Security and Medicare portions because the reference calculation taxes only the stated self-employment income after the 92.35% adjustment." -us,scenario_089,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly treated the entire $19,650 partnership or S-corp income entry as subject to self-employment tax and added it to the $30,335 self-employment income. Its final value is exactly the result of applying the 92.35% adjustment and 15.3% rate to that overstated $49,985 base." -us,scenario_089,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's revised inputs nearly reached the correct $80,700.13 taxable-income base and its own arithmetic produced about $3,230, but it submitted $4,022 instead. The submitted value does not follow from any calculation in its explanation." -us,scenario_089,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model never applied the specified NC computation: $106,200.13 AGI minus the $25,500 joint standard deduction, taxed at 3.99%. Its $4,532 answer reflects an overstated tax base or rate and incorrectly frames unspecified federal credits as relevant to this no-child NC liability." -us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's own final computation yielded roughly $3,004 after incorrectly deducting the head's unsupported 401(k) contribution, yet it submitted $4,710. The submitted value is disconnected from its stated income base, deduction, and rate." -us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly reconstructed approximately $80,699 of NC taxable income and approximately $3,220 of tax, then abandoned that derivation and invented a $132,500 taxable base. That unexplained recomputation produced the erroneous $5,290 submission." -us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model stated an AGI near $106,500 and the correct $25,500 deduction but then used $87,100 as taxable income. Subtracting the deduction yields about $81,000, not $87,100, so it overstated the 3.99% tax base." -us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model replaced the 2026 NC parameters with a 4.25% rate and a $30,000 joint standard deduction. The applicable computation uses the 3.99% rate and $25,500 deduction on $106,200.13 of AGI." -us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model improperly multiplied all $49,985 of self-employment and partnership/S-corporation income by 92.35% and calculated the half-SE-tax deduction on that combined amount. The trace instead produces $106,200.13 of AGI; the model then submitted $3,800 despite its own erroneous calculation yielding about $2,979." -us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI as $104,999.90 instead of $106,200.13. With the correct $25,500 deduction and 3.99% rate, taxable income is $80,700.13 and tax is $3,219.94." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI by using $98,411 rather than $106,200.13. This reduced NC taxable income from $80,700.13 to $72,911 before applying the otherwise correct deduction and rate." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied no tax base, deduction, or rate supporting $4,410. The correct derivation taxes $80,700.13 at 3.99%, and $4,410 implies an unsupported taxable base of about $110,526 at that rate." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI as $98,411 instead of $106,200.13. Consequently, it taxed $72,911 rather than the correct $80,700.13 after the $25,500 joint deduction." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model both understated AGI as $98,411 and used an erroneous $29,250 NC joint standard deduction. The correct inputs are $106,200.13 of AGI and a $25,500 deduction, yielding $80,700.13 of taxable income." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated NC AGI as $98,410.90 rather than $106,200.13. Its deduction and rate were correct, but the erroneous AGI reduced taxable income by $7,789.23." -us,scenario_089,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced self-employment income to 92.35% before also subtracting the half-SE-tax deduction, double-counting the SE-tax adjustment in AGI. NC AGI is $106,200.13, not $103,879.36." -us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the household's substantial income as eliminated by deductions and returned zero. The $106,200.13 AGI exceeds the $25,500 joint standard deduction by $80,700.13, which is taxable at 3.99%." -us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The submitted $3,055 at the 3.99% rate implies taxable income of about $76,566, reflecting roughly $4,134 of unsupported additional deductions. The correct taxable base after the joint standard deduction is $80,700.13." -us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted employer-sponsored insurance premiums from the separately stated gross wages, driving AGI down to $98,411. The trace uses $106,200.13 of AGI, followed only by the $25,500 NC standard deduction in computing state taxable income." -us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $69,696 rather than $80,700.13. Its answer therefore embeds $11,004.13 of unsupported additional income reductions beyond the $25,500 joint standard deduction." -us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"The stated method should produce $3,219.94 from $106,200.13 of AGI, the $25,500 deduction, and the 3.99% rate. The $3,230 submission reflects an arithmetic or input-rounding error inconsistent with that exact derivation." -us,scenario_089,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"At the 3.99% rate, $3,120 implies taxable income of about $78,195 rather than $80,700.13. The model therefore removed about $2,505 of unsupported income beyond the $25,500 joint standard deduction." -us,scenario_089,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a 4.25% tax rate on the otherwise correct approximate taxable-income base. North Carolina's applicable 2026 rate is 3.99%, which applied to $80,700.13 produces $3,219.94." -us,scenario_089,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $103,664 and taxable income as $78,164 instead of $106,200.13 and $80,700.13. It also described the rate as 4% rather than applying the exact 3.99% rate." -us,scenario_089,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so it failed the required submission contract." -us,scenario_089,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI as $102,732.90 rather than $106,200.13. This caused it to tax $77,232.90 instead of $80,700.13 after the correct $25,500 deduction." -us,scenario_089,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used a 4.5% NC rate instead of the applicable 3.99% rate and also understated AGI as about $99,409. The correct calculation applies 3.99% to $80,700.13." -us,scenario_089,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a 4.5% NC rate instead of 3.99%, added the separately listed FLSA overtime premium to annual gross wages even though annual wages include overtime, and deducted the head's 401(k) contribution despite no head wages. It also calculated self-employment tax using partnership/S-corporation income as part of the SE-tax base, further distorting AGI." +us,scenario_086,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly calculated $3,844 of Social Security tax and $899 of Medicare tax, then incorrectly added $466.80 as mandatory employee Georgia unemployment insurance tax. Georgia unemployment insurance is an employer tax and is excluded from employee-side payroll tax, so the two valid components total $4,743." +us,scenario_086,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model reduced FICA-taxable wages from $62,000 to $60,440 by subtracting the separately listed $1,560 health-insurance premium. The benchmark applies Social Security and Medicare taxes to the full $62,000 wage amount, yielding $3,844 plus $899, or $4,743." +us,scenario_086,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the entire $19,648 employer-sponsored insurance premium as a pre-tax employee salary deduction and reduced FICA wages to $42,352. That premium is not an employee wage-reduction contribution in the supplied facts, so FICA applies to the full $62,000 and totals $4,743." +us,scenario_086,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model misclassified the $19,648 employer-sponsored insurance premium as a pre-tax employee-paid contribution and subtracted it from wages. The payroll-tax base remains $62,000, producing $3,844 of Social Security tax and $899 of Medicare tax, totaling $4,743." +us,scenario_086,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct $3,844 Social Security component and a 1.45% Medicare calculation on $62,000, which equals $899, but failed to add them correctly and submitted $3,979. Those stated components sum to $4,743, with no Georgia employee payroll component or Additional Medicare Tax." +us,scenario_086,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." +us,scenario_086,payroll_tax,qwen3.8-max,llm_error,other,False,"The model correctly identified $3,844 of Social Security tax and $899 of Medicare tax but submitted $3,813 instead of summing its own components. The correct arithmetic is $3,844 + $899 = $4,743." +us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used an invented $24,000 head-of-household deduction and a $4,000 dependent exemption instead of the $15,000 standard deduction and $5,000 dependent exemption. It also submitted $2,732 despite its own recomputation yielding $1,749.03, so its final value does not follow its stated calculation." +us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model applied Georgia's obsolete graduated 1%–6% brackets and treated the filer as single. The 2026 computation uses the applicable Georgia schedule for a head-of-household filer after reducing $61,662.50 of Georgia AGI by $20,000 of deductions and exemptions." +us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model substituted an $18,500 standard deduction and $4,000 dependent exemption for the applicable $15,000 and $5,000 amounts, producing the wrong taxable-income base. Its submitted $2,191 also contradicts its own stated multiplication result of $2,032.51." +us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used a $12,000 standard deduction and $4,000 dependent exemption instead of $15,000 and $5,000, and applied the wrong rate. Its stated arithmetic yields $2,324, but it then reduced the result to $1,869 through unspecified ""personal exemption considerations"" that were not part of its taxable-income calculation." +us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the entire $12,824 Social Security benefit from an AGI that had already excluded it, rather than subtracting only the $10,900.40 taxable portion from federal AGI of $72,562.90. It then used obsolete $5,400, $2,700, and $3,000 deduction and exemption amounts instead of the $15,000 head-of-household deduction and $5,000 dependent exemption." +us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the taxable Social Security amount from federal AGI, then used a $12,000 standard deduction and $4,000 dependent exemption instead of $15,000 and $5,000. Its submitted $2,765 also does not follow its own stated estimate of about $2,372." +us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model started from an unsupported AGI of $60,140 and used obsolete $5,400 standard-deduction and $5,400 personal-exemption amounts. Georgia taxable income is $41,662.50 after subtracting $15,000 and $5,000 from Georgia AGI of $61,662.50, not $49,340." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented federal AGI of $52,952.40 and treated a $15,800 personal exemption as the principal Georgia deduction. The correct Georgia AGI is $61,662.50, followed by a $15,000 standard deduction and $5,000 dependent exemption." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies a rough rate-on-wages calculation rather than the required state tax-base derivation. It failed to derive $61,662.50 of Georgia AGI, subtract the $15,000 standard deduction and $5,000 dependent exemption, and tax the resulting $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model correctly reached Georgia AGI near $61,662 but used a $12,000 standard deduction and $3,000 dependent deduction instead of $15,000 and $5,000. That overstated taxable income by $5,000 and led to excessive tax." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an unsupported federal AGI of $52,952 and taxable income of about $37,952. The trace yields Georgia AGI of $61,662.50 and taxable income of $41,662.50 after the specified $15,000 standard deduction and $5,000 dependent exemption." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The submitted $3,047 implies that the model taxed a substantially overstated base despite claiming to apply deductions and exemptions. The required base is $41,662.50 after subtracting $20,000 from Georgia AGI of $61,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model correctly recognized the $10,900.40 taxable Social Security subtraction but allowed only a $10,600 head-of-household deduction and no $5,000 dependent exemption. This produced taxable income of $51,062 instead of $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer gives no derivation and its low amount implies excessive deductions or an understated taxable-income base. The prescribed deductions reduce Georgia AGI of $61,662.50 only to $41,662.50, whose tax is $2,078.96." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model imported federal-style standard and medical deductions into the Georgia calculation and reduced the liability to $498 without showing a valid state tax base. Georgia taxable income is $41,662.50 after the $15,000 standard deduction and $5,000 dependent exemption; the listed medical expenses do not replace that derivation." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model omitted taxable Social Security from federal AGI and used a $12,000 standard deduction plus only $4,500 of exemptions. The correct sequence produces Georgia AGI of $61,662.50 and taxable income of $41,662.50 after $20,000 of deductions and exemptions." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $1,357.80 result implies that it deducted substantially more than the applicable $20,000 from Georgia AGI. Excluding the $10,900.40 taxable Social Security amount leaves $61,662.50, and the $15,000 standard deduction plus $5,000 dependent exemption leave $41,662.50 taxable." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used $12,000 and $4,000 deductions, leaving $45,662.40, instead of the $15,000 standard deduction and $5,000 dependent exemption that leave $41,662.50. It also applied an incorrect 5.09% rate." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model stated the Social Security subtraction and filing-status adjustments only generically, but its $1,903 result reflects an understated tax base or excessive deductions. The applicable deductions total $20,000, leaving $41,662.50 taxable and $2,078.96 of tax." +us,scenario_086,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model supplied no computation supporting $2,184 beyond a generic assertion about deductions and exemptions. The complete calculation taxes $41,662.50 after the Social Security subtraction, $15,000 standard deduction, and $5,000 dependent exemption, producing $2,078.96." +us,scenario_086,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a single $18,000 head-of-household deduction and omitted the separate $5,000 dependent exemption. The applicable $15,000 standard deduction plus $5,000 dependent exemption leave taxable income of $41,662.50, not $43,662." +us,scenario_086,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted the $338 educator expense after starting from federal AGI and used obsolete $5,400 standard-deduction and $5,400 exemption amounts. The correct state deductions are $15,000 and $5,000, producing taxable income of $41,662.50 rather than $50,514." +us,scenario_086,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model approximated taxable income as $40,700 rather than carrying through Georgia AGI of $61,662.50 less the exact $15,000 standard deduction and $5,000 dependent exemption. That exact sequence leaves $41,662.50 taxable and produces $2,078.96." +us,scenario_086,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model used a $12,000 standard deduction and two $2,700 personal exemptions instead of the $15,000 head-of-household standard deduction and $5,000 dependent exemption. It also applied an incorrect 4.99% rate, yielding tax on $44,262 rather than the applicable calculation on $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model understated Georgia AGI as $61,387 and used an $18,500 standard deduction plus $3,000 dependent exemption. Georgia AGI is $61,662.50, and the applicable $15,000 and $5,000 reductions leave $41,662.50 taxable." +us,scenario_086,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model began from wages alone, used obsolete $5,000 and $2,700 deductions, and omitted the dependent exemption. It should begin with federal AGI including taxable Social Security, subtract the Georgia Social Security subtraction, and then apply the $15,000 standard deduction and $5,000 dependent exemption." +us,scenario_086,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $12,824 Social Security benefit as Georgia income, even though Georgia subtracts the $10,900.40 taxable portion included in federal AGI. It also used $12,000, $2,700, and $3,000 deductions instead of the applicable $15,000 standard deduction and $5,000 dependent exemption, and applied an incorrect rate." +us,scenario_086,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model's arithmetic is internally inconsistent: excluding Social Security from $74,824 and then subtracting $23,100 cannot yield $51,700. The correct derivation reaches Georgia AGI of $61,662.50 and subtracts $20,000, leaving taxable income of $41,662.50." +us,scenario_086,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model invented a Georgia refundable dependent-child credit of $260 per qualifying child. Georgia has no such applicable refundable credit, so the 12-year-old dependent generates $0 in state refundable credits." +us,scenario_088,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the deduction for the $9,600 of qualified tips included in wages, leaving $7,085 of purported taxable income. It also incorrectly included the $675 state tax refund in gross income without establishing a prior-year itemized-deduction tax benefit." +us,scenario_088,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied neither a numeric value nor an explanation for the requested output, violating the required output contract." +us,scenario_088,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model's reasoning arrived at approximately zero tax but submitted -$3. Federal income tax before refundable credits is floored at zero, and the qualified-tip deduction reduces taxable income sufficiently to produce exactly $0; it also incorrectly described an age-based additional standard deduction at age 56, when that addition begins at age 65." +us,scenario_088,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the state tax refund and reached AGI of about $23,360, but it omitted the deduction for the $9,600 of qualified tips included in wages. Applying that deduction alongside the standard deduction eliminates the $6,810 of taxable income on which it computed $681 of tax." +us,scenario_088,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model applied the childless EITC phaseout using $18,577 of wages instead of the greater applicable amount, the household's approximately $24,035 AGI. That AGI fully phases out the credit; the model also contradicted its own phaseout arithmetic by jumping from an estimated $76 credit to $632." +us,scenario_088,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output or reasoning for federal_refundable_credits, violating the required structured-output contract." +us,scenario_088,federal_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model calculated the childless EITC phaseout solely from wages of $18,577. The phaseout instead uses the greater of earned income and AGI, and the approximately $24,035 AGI reduces the EITC to $0." +us,scenario_088,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated gross earned income below the wage-based limit as sufficient for a childless EITC and omitted the AGI phaseout test. Approximately $24,035 of AGI exceeds the single no-child limit and yields no EITC." +us,scenario_088,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model compared only $18,577 of earned income with its estimated childless EITC cutoff and ignored the rule that the phaseout uses the greater of earned income and AGI. The household's approximately $24,035 AGI fully phases out the EITC, leaving federal refundable credits at $0." +us,scenario_088,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated stated disability as sufficient for the aged, blind, or disabled pathway without applying its financial and categorical requirements. At roughly $23,375 of income and with no SSI receipt, the head qualifies through no Medicaid category." +us,scenario_088,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model calculated SSI-style countable income of $13,561.50, recognized that it exceeded the approximately $11,604 SSI-linked limit, and then discarded that result to award eligibility based on disability and blindness alone. Those statuses do not create automatic SSI-related Medicaid eligibility, and the head receives no SSI." +us,scenario_088,head_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"The model asserted that $31,776 in bank assets and the head’s countable income were within typical SSI-related Medicaid limits without applying the actual financial test. Disability and blindness only open the ABD pathway for evaluation; these facts do not satisfy it, and no other Medicaid category applies." +us,scenario_088,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model used a blanket disabled-plus-low-income shortcut instead of testing a recognized Medicaid pathway. Disability alone does not confer eligibility, and MAGI income at 1.46 times FPL does not qualify this Texas head through a MAGI category." +us,scenario_088,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model invented a Medicaid eligibility pathway based on housing assistance and characterized Texas as expanding access for this disabled adult. Section 8 or other housing assistance is not a Medicaid eligibility category, and disability alone does not satisfy an ABD or SSI-related pathway." +us,scenario_088,head_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated a disability flag as sufficient for Medicare eligibility at any age. It failed to require the pre-65 disability pathway's SSDI entitlement and 24-month waiting period, neither of which is present." +us,scenario_088,head_medicare_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model correctly identified the SSDI-based 24-month pathway but then inferred SSDI receipt and completion of the waiting period solely from the disability flag. Those unlisted conditions are false, so the 56-year-old head does not qualify through that pathway." +us,scenario_088,head_medicare_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that disability qualifies a person for Medicare regardless of age. Before age 65, disability must connect to a qualifying Medicare pathway such as SSDI entitlement for the required period, which the household facts do not establish." +us,scenario_088,head_medicare_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model invented a PolicyEngine rule making the disability boolean alone sufficient for Medicare eligibility under age 65. The head is 56 and has no listed SSDI entitlement, completed waiting period, ESRD, or other pre-65 qualifying condition." +us,scenario_088,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly separated tips from gross wages even though the prompt states that the $9,600 of tips is already included in the $18,577 wage total, then invented an employee tip-credit reduction. All $18,577 is subject to both employee Social Security and Medicare taxes, with no employee tip credit." +us,scenario_088,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model excluded the explicitly reported $18,577 of wages from the employee payroll-tax base. Those wages generate both employee Social Security tax and employee Medicare tax even though Texas imposes no separate mandatory employee state payroll tax." +us,scenario_088,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model identified the correct 6.2% Social Security and 1.45% Medicare rates and the correct $18,577 wage base but performed the arithmetic incorrectly. Applying the combined 7.65% rate yields $1,421.14, not $1,270." +us,scenario_088,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract." +us,scenario_088,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model performed the disabled-household net-income test and an uncapped excess-shelter deduction but never applied the SNAP countable-resource test to the head’s $31,776 bank balance. Those liquid assets exceed the applicable Texas SNAP resource limit, so the household is ineligible and the computed $159 monthly allotment never arises." +us,scenario_088,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model calculated a benefit from income and shelter deductions while omitting the $31,776 bank account from SNAP eligibility. That countable resource balance exceeds the applicable limit for a household with a disabled member, ending eligibility before its estimated $100 monthly benefit calculation; it also incorrectly applied a shelter-deduction cap despite the disabled member." +us,scenario_088,snap,glm-5.2,llm_error,asset_resource,False,"The model treated high mortgage interest and property taxes as reducing net income to zero and then awarded the maximum allotment, without testing the $31,776 bank balance against the SNAP resource limit. The excess countable resources make the household ineligible regardless of the shelter deduction." +us,scenario_088,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model moved directly from the disabled-member net-income pathway and large shelter deduction to the maximum one-person allotment, omitting the SNAP asset test. The head’s $31,776 in bank-account assets exceeds the applicable countable-resource limit, so no allotment is payable." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $3,467.43 overtime income deduction, leaving taxable income at about $64,432 instead of $60,964.27. Its submitted $8,225 also contradicts its own stated regular-tax calculation of about $7,236." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented $57,893 of wages for the head even though no head wage or hourly rate was listed and unlisted income had to be zero. It also omitted the QBI and overtime deductions and applied unsupported adjustments for insurance premiums." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model approximated the standard deduction and QBI deduction, omitted the $3,467.43 overtime deduction, and then submitted $9,089 despite deriving regular income tax of about $7,377. The correct deductions reduce taxable income to $60,964.27." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model added $4,286 of self-employment tax to regular federal income tax. Self-employment tax is a separate requested output and is excluded from federal_income_tax_before_refundable_credits; only its deductible half enters the AGI calculation." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's intermediate work placed regular income tax near $7,150, then submitted $12,850 without a valid computation. It also omitted the $3,467.43 overtime deduction and used approximate deduction parameters instead of taxable income of $60,964.27." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the head's $5,403 desired traditional 401(k) contribution despite the head having no wages; PolicyEngine applies only the spouse's $6,946 deferral. It also omitted the $3,467.43 overtime deduction and used a $30,800 standard deduction rather than $32,200, with partially offsetting errors producing $6,761." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model treated the $19,650 partnership/S-corp income as self-employment income, inflating self-employment tax and its half-tax deduction, and omitted the QBI and overtime deductions. It therefore never reached taxable income of $60,964.27." +us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated taxable wages, used the obsolete $29,200 standard deduction, calculated QBI as 20% of unreduced business income, and omitted the overtime deduction. The QBI base is $47,842.18 and the applicable deductions total $45,235.87." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA individual provisions expired for 2026 and substituted a $16,000 standard deduction, personal exemptions, and a 15% bracket. It also subtracted employer-sponsored insurance premiums again even though the traced taxable wage amount is $58,934.98 after the applicable pretax 401(k) deferral." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer did not account for the traced $9,568.44 QBI deduction and $3,467.43 overtime deduction, and it did not identify the applicable $32,200 standard deduction. Those deductions reduce taxable income to $60,964.27 and tax to $6,819.71." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model wrongly applied a supposed TCJA sunset, personal exemptions, pre-TCJA brackets, and a $16,600 standard deduction. It also double-subtracted $7,789 of employer insurance premiums and omitted both the QBI and overtime deductions." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used pre-TCJA personal exemptions, a $15,300 standard deduction, and a 15% bracket instead of the applicable 2026 rules. It also double-subtracted employer insurance premiums and omitted the QBI and overtime deductions." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $7,789 of employer-sponsored insurance premiums from the stated wages, reducing AGI to $98,410.90 instead of $106,200.13. It also omitted the $3,467.43 overtime deduction, so its taxable-income calculation was understated overall." +us,scenario_089,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $30,600 standard deduction instead of $32,200 and omitted the $3,467.43 overtime deduction. These errors left taxable income at $66,031.53 rather than $60,964.27." +us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model ignored the instruction that the two adults form one household tax group, filing and take-up are assumed, and unlisted inputs are zero. The listed income and deductions fully determine a joint-filer taxable income of $60,964.27 and positive tax of $6,819.71." +us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model's near miss reflects an approximate bracket computation rather than the exact 2026 schedule applied to taxable income of $60,964.27. That schedule yields $6,819.71, not $6,828." +us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model reduced wages for unspecified pretax health deductions and arrived at taxable income of about $62,996 instead of $60,964.27. The traced calculation uses AGI of $106,200.13 and deductions of $32,200, $9,568.44, and $3,467.43." +us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model named several deductions but omitted the separate $3,467.43 overtime deduction from its stated computation. It therefore failed to reduce taxable income to $60,964.27 before applying the 2026 brackets." +us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used an unsupported combined-income shortcut near $120,000 and did not compute AGI, the QBI deduction, or the overtime deduction. The correct path produces $106,200.13 of AGI and $60,964.27 of taxable income." +us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model wrongly applied a TCJA sunset, personal exemptions, a $16,637 standard deduction, and a 15% bracket. The applicable 2026 computation instead uses the $32,200 standard deduction, the QBI and overtime deductions, and the current 10%/12% bracket schedule." +us,scenario_089,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted the $3,467 overtime premium on top of gross wages, deducted the head's unfunded 401(k) contribution, and invented a $600 self-employed health-insurance deduction. It also omitted the QBI and overtime income deductions from taxable income." +us,scenario_089,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model calculated QBI as 20% of the unreduced $49,985 business income instead of 20% of $47,842.18 after the allocable half-SE-tax deduction. It also omitted the $3,467.43 overtime deduction, leaving taxable income above $60,964.27." +us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model added the $3,467 FLSA overtime premium to wages even though gross wages already include overtime, then deducted both spouses' desired 401(k) contributions despite the head having no wages. It also omitted the $9,568.44 QBI deduction and the $3,467.43 overtime income deduction." +us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly reconstructed taxable income to within rounding of $60,964.27 but applied an inexact 2026 bracket threshold or arithmetic. Exact application of the 2026 married-filing-jointly rates yields $6,819.71 rather than $6,827.74." +us,scenario_089,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model deducted both spouses' desired 401(k) contributions even though the head had no wages and computed half of self-employment tax as $3,531 instead of $2,143.10. It also used an unreduced QBI base, an obsolete standard deduction, and omitted the overtime deduction." +us,scenario_089,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the head's $5,403 desired 401(k) contribution despite zero head wages and omitted both the $9,568.44 QBI deduction and $3,467.43 overtime deduction. Its bracket calculation also used an incorrect base amount for the 10% bracket." +us,scenario_089,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model invented $132,619 of head wages by applying the spouse's $27 hourly rate and overtime facts to the head, then invented four children and corresponding tax credits. No head wages or dependents were listed, so those additions violate the zero-for-unlisted-input rule and destroy the tax calculation." +us,scenario_089,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model awarded a $2,242.80 Additional Child Tax Credit despite the household having no qualifying children. It also fabricated $120,000 of head wages from hours worked and mislabeled $2,400 of supposed excess Additional Medicare withholding as a refundable credit, even though no head wage or hourly rate was listed and tax withholding refunds are not included in federal_refundable_credits." +us,scenario_089,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model invented wage income for the head, added estimated North Carolina payroll taxes that do not exist, and initially conflated the spouse’s self-employment and pass-through income with W-2 wages. Employee FICA applies only to the spouse’s $65,881 of wages here, producing $5,039.91." +us,scenario_089,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model identified the correct $65,881 employee-FICA wage base but made arithmetic errors in Medicare tax and then submitted $5,085.44, a number unsupported by its own calculations. The engine’s unrounded components are $4,084.63 of Social Security tax and $955.28 of Medicare tax, totaling $5,039.91." +us,scenario_089,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model incorrectly deducted $1,200 of separately reported health premiums from FICA wages. Those inputs do not reduce the spouse’s $65,881 payroll-tax base, so Social Security and Medicare tax total $5,039.91." +us,scenario_089,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $7,789 employer-sponsored insurance premium from gross wages when constructing the FICA base. PolicyEngine applies employee Social Security and Medicare taxes to the full $65,881 of wages, yielding $5,039.91." +us,scenario_089,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model double-counted the $3,467 FLSA overtime premium by adding it to the $65,881 annual gross-wage total, even though annual gross wages already include overtime. The employee-FICA base remains $65,881, not $69,348." +us,scenario_089,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly treated the $7,789 employer-sponsored insurance premium as a pre-tax reduction to the spouse’s FICA wages. The full $65,881 wage amount is subject to employee Social Security and Medicare taxes, totaling $5,039.91." +us,scenario_089,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA-taxable wages by the $7,789 employer-sponsored insurance premium. That input does not reduce the payroll-tax wage base, so FICA applies to all $65,881 of wages." +us,scenario_089,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $7,789 employer-sponsored insurance premium from gross wages for FICA purposes. Employee Social Security and Medicare taxes apply to the spouse’s full $65,881 wage amount and total $5,039.91." +us,scenario_089,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct base—only the spouse’s $65,881 of wages—but submitted $8,107 instead of applying the combined 7.65% employee FICA rate. The stated method yields $5,039.91 using the engine’s unrounded Social Security and Medicare components." +us,scenario_089,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model applied FICA to wages for both spouses even though the head has no listed wage income. Only the spouse’s $65,881 is subject to employee payroll tax, yielding $5,039.91." +us,scenario_089,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model incorrectly excluded $7,789 of employee health premiums from the FICA wage base. PolicyEngine taxes the full $65,881 of wages for Social Security and Medicare, producing $5,039.91." +us,scenario_089,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The $8,120 answer is inconsistent with applying employee FICA to the spouse’s $65,881 annual wages and indicates that additional income or overtime was added to the payroll-tax base. The overtime premium is already included in gross wages, and employee FICA on the proper base totals $5,039.91." +us,scenario_089,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model double-counted the $3,467 overtime premium by adding it to the annual gross-wage figure and taxing $69,348. Because gross wages already include overtime, FICA applies to $65,881 and totals $5,039.91." +us,scenario_089,payroll_tax,inkling,llm_error,other,False,"The model rounded the Social Security component down to $4,083 instead of calculating 6.2% of the wage base accurately, then submitted an approximate total. PolicyEngine’s unrounded calculation produces $4,084.63 of Social Security tax plus $955.28 of Medicare tax, totaling $5,039.91." +us,scenario_089,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model added half of the spouse’s self-employment tax to the employee payroll-tax output. Self-employment tax is a separate requested quantity and none of it belongs in payroll_tax; this output contains only the $5,039.91 employee FICA liability on wages." +us,scenario_089,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model’s $10,068.71 Social Security component applies the 6.2% rate to an invented base far above the spouse’s $65,881 of wages, thereby including income that is not employee wages. With no head wages and with self-employment and pass-through income excluded, Social Security and Medicare taxes total $5,039.91." +us,scenario_089,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model selected the correct $30,335 base and stated the correct 92.35% and 15.3% factors, but calculated the adjusted earnings incorrectly and then submitted $4,275.23 despite its own intermediate result of $4,285.34. Applying the components as PolicyEngine does yields $4,286.24." +us,scenario_089,self_employment_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model correctly excluded the separately listed partnership-or-S-corporation income, but multiplied $30,335 by 92.35% incorrectly, reporting $28,024.32 instead of $28,014.37. That arithmetic error propagated into its overstated tax." +us,scenario_089,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model incorrectly added the $19,650 partnership-or-S-corporation income to the explicit $30,335 self-employment-income input. PolicyEngine's self-employment-tax base uses the latter input, and the model's submitted $6,355 also contradicts its own $7,062.66 calculation." +us,scenario_089,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model named the correct calculation—15.3% of 92.35% of $30,335—but its $4,357 result does not equal that calculation. The engine's component calculation yields $4,286.24." +us,scenario_089,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model correctly said the $19,650 partnership-or-S-corporation amount was excluded, but $5,114 is inconsistent with applying the self-employment-tax formula to $30,335. The 92.35% adjustment followed by the Social Security and Medicare components yields $4,286.24." +us,scenario_089,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model stated the correct $30,335 base, 92.35% adjustment, and 15.3% rate but submitted $3,677, which does not follow from those inputs. It also invoked Additional Medicare Tax irrelevantly; that tax is excluded from the requested self-employment-tax output." +us,scenario_089,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model described the base as including partnership income, which is not the base used by PolicyEngine for this input, and then supplied only a rough unsupported estimate. Applying the engine's self-employment-tax components to the explicit $30,335 self-employment income yields $4,286.24, not $4,280." +us,scenario_089,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output or explanation, violating the required output contract." +us,scenario_089,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model incorrectly treated all $19,650 of the combined partnership-or-S-corporation income field as additional self-employment earnings. PolicyEngine applies self-employment tax to the separately specified $30,335 self-employment-income input, producing $4,286.24." +us,scenario_089,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model recognized that S-corporation income is not subject to self-employment tax but nevertheless classified the entire ambiguous combined partnership-or-S-corporation amount as partnership income and added it to the base. The engine uses the explicit $30,335 self-employment-income input and derives $4,286.24." +us,scenario_089,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model incorrectly combined the $30,335 self-employment income with the $19,650 partnership-or-S-corporation amount and then invented a $668.78 offset for employee Social Security tax. W-2 Social Security withholding does not create a dollar-for-dollar offset against self-employment tax; wages only reduce the remaining Social Security wage-base capacity, which is not exhausted here." +us,scenario_089,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's reasoning actually reached approximately the correct $106,200 AGI and a tax near $3,220, but it submitted $4,022 instead. The submitted value does not follow its stated $80,952 taxable-income base or 3.99% rate." +us,scenario_089,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model never completed the required calculation from $106,200.13 of AGI through the $25,500 joint standard deduction and instead submitted an unsupported $4,532. Correct taxable income is $80,700.13, producing $3,219.94 at 3.99%." +us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model computed estimates between $3,004 and $3,200 but then submitted $4,710 without a corresponding tax base or rate. It also improperly deducted the head's unavailable 401(k) contribution, understating AGI in its scratch calculation." +us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model derived $80,699 of taxable income and approximately $3,220 of tax, then abandoned that calculation and submitted $5,290 based on an invented $132,500 taxable-income base. The correct trace uses $80,700.13 at 3.99%." +us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model inflated North Carolina taxable income from $80,700.13 to about $87,100 without identifying any valid state addition. North Carolina applies 3.99% to the AGI-minus-standard-deduction base, yielding $3,219.94." +us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a 4.25% rate and a $30,000 standard deduction instead of the applicable 3.99% rate and $25,500 joint standard deduction. It also deducted both traditional IRA contributions rather than following the traced AGI of $106,200.13." +us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly multiplied the combined self-employment and partnership income by 92.35% and treated the entire amount as subject to the self-employment-tax adjustment. Partnership or S-corporation income remains a separate $19,650 AGI component here, and the correct AGI is $106,200.13." +us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI by $1,200.23, using $104,999.90 instead of $106,200.13. With the same $25,500 deduction and 3.99% rate, that incorrect AGI directly caused the $48.69 shortfall." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI by $7,789.13, using $98,411 instead of $106,200.13. That difference matches an improper extra deduction of one $7,789 employer-sponsored insurance premium from the income base." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $4,410 implies taxable income of about $110,526 at 3.99%, rather than the traced $80,700.13. The model failed to apply the correct $106,200.13 AGI and $25,500 joint standard deduction." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI by $7,789.13, using $98,411 instead of $106,200.13. It therefore applied 3.99% to $72,911 rather than the correct $80,700.13 taxable income." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model combined an understated $98,411 AGI with an inflated $29,250 joint standard deduction. The applicable inputs are $106,200.13 of AGI and a $25,500 deduction, producing $80,700.13 of taxable income." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated AGI by $7,789.23, using $98,410.90 instead of $106,200.13. This is consistent with deducting one listed $7,789 employer-sponsored insurance premium even though the trace does not subtract it from AGI." +us,scenario_089,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model reduced self-employment income to 92.35% and then also subtracted half of self-employment tax, double-counting the 7.65% adjustment in the AGI calculation. It also deducted both IRA contributions, leading to $103,879.36 rather than the traced $106,200.13 AGI." +us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the household's taxable income as eliminated by deductions despite $106,200.13 of AGI and only a $25,500 North Carolina standard deduction. The remaining $80,700.13 is taxable at 3.99%, so the liability is not zero." +us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model submitted an unsupported estimate after unspecified retirement deductions and adjustments. The required calculation is $106,200.13 minus $25,500, followed by 3.99%, which yields $3,219.94 rather than $3,055." +us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted a $7,789 employer-sponsored insurance premium from wages, reducing AGI to about $98,411. The traced AGI is $106,200.13, so taxable income is $80,700.13 rather than $72,911." +us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $69,696, understating the traced $80,700.13 base by $11,004.13. No identified North Carolina adjustment supports that additional reduction beyond the $25,500 joint standard deduction." +us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"The model described the correct 3.99% framework but rounded an exact $3,219.94 calculation to $3,230. Applying the rate to $80,700.13 yields the requested amount without that ten-dollar estimation error." +us,scenario_089,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The terse $3,120 answer implies a taxable-income base below $80,700.13 at the applicable 3.99% rate. The model failed to use the traced $106,200.13 AGI less the $25,500 joint standard deduction." +us,scenario_089,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a 4.25% rate instead of North Carolina's applicable 2026 rate of 3.99%. Its AGI and standard deduction were essentially correct, so the wrong rate alone raised the result from $3,219.94 to about $3,430." +us,scenario_089,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated AGI at $103,664 instead of $106,200.13 and rounded the rate to 4% instead of using 3.99%. The correct taxable base is $80,700.13, producing $3,219.94." +us,scenario_089,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an unsupported standard deduction of about $26,700 instead of $25,500. Starting from its nearly correct $106,200 AGI, the applicable deduction leaves $80,700.13 taxable and produces $3,219.94." +us,scenario_089,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required structured-output contract before any substantive tax calculation could be evaluated. +us,scenario_089,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI by $3,467.23, using $102,732.90 rather than $106,200.13. The difference corresponds almost exactly to the listed $3,467 FLSA overtime premium, which should not have been subtracted from the annual gross-wage input." +us,scenario_089,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used a 4.5% rate instead of the applicable 2026 North Carolina rate of 3.99%. It also understated AGI at about $99,409, compounding the rate error with an incorrect taxable-income base." +us,scenario_089,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a 4.5% North Carolina rate instead of 3.99% and incorrectly added the $3,467 FLSA overtime premium on top of annual gross wages that already include overtime. It also subjected the partnership or S-corporation income to the self-employment-tax adjustment and deducted the head's unavailable 401(k) contribution." +us,scenario_089,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented $248,485 of AGI, a $32,200 standard deduction, and an out-of-state tax credit unsupported by any household fact. The actual calculation uses $106,200.13 of AGI, a $25,500 deduction, and no such credit, yielding $3,219.94." us,scenario_089,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_090,federal_refundable_credits,claude-opus-4.8,llm_error,age_disability,False,"The model treated the 22-year-old as eligible for the childless EITC, overlooking its minimum age of 25. Its subsequent phase-in and phaseout estimate is irrelevant because age disqualifies the taxpayer, producing $0." -us,scenario_090,federal_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"The model applied the childless EITC schedule without enforcing the minimum claimant age of 25. At age 22, the head receives no EITC, and the absence of children or other qualifying circumstances leaves total refundable credits at $0." -us,scenario_090,federal_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model multiplied wages by the 7.65% childless EITC phase-in rate before checking age eligibility. The head is 22 and therefore fails the age-25 minimum, so the EITC is $0." -us,scenario_090,federal_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"The model incorrectly awarded the childless EITC to a 22-year-old. Because a filer without qualifying children must be at least 25, the 7.65% calculation does not apply and refundable credits equal $0." -us,scenario_090,federal_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"The model explicitly declared the 22-year-old eligible for the childless EITC, missing the minimum age of 25. Income being below the phaseout threshold cannot cure that categorical age failure." -us,scenario_090,federal_refundable_credits,grok-4.3,llm_error,age_disability,False,"The model labeled the amount a full childless EITC without applying the minimum age requirement. A 22-year-old with no qualifying children is ineligible, so no EITC or other refundable federal credit is available." -us,scenario_090,federal_refundable_credits,grok-4.5,llm_error,age_disability,False,"The model applied the 7.65% phase-in rate and rounded the result while omitting the childless EITC age test. The head is younger than 25, making the credit $0 before any rate, threshold, or rounding calculation." -us,scenario_090,federal_refundable_credits,minimax-m3,llm_error,age_disability,False,The model focused on earnings and investment-income thresholds but failed to enforce the childless EITC minimum age of 25. The $20 of interest is immaterial because the 22-year-old is categorically ineligible for the credit. -us,scenario_090,federal_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model calculated and then arbitrarily adjusted a childless EITC amount without first applying the minimum age requirement. At age 22, the head cannot claim the childless EITC, so neither $535.50 nor $492 is available and total refundable credits are $0." -us,scenario_090,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly stated that Kansas adopted Medicaid expansion and applied the 138% FPL expansion-adult threshold. Kansas has no expansion pathway for this 22-year-old childless adult, so low MAGI does not establish eligibility; the asset discussion is irrelevant because no eligibility category applies." -us,scenario_090,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented a Kansas adult Medicaid pathway based on being under age 26. The age-26 rule concerns dependent private-plan coverage, not Medicaid eligibility, and this 22-year-old nondependent adult falls into no Medicaid category despite income at 0.44 FPL." -us,scenario_090,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated extremely low income as sufficient for Medicaid in a non-expansion state. At age 22, the head is outside the age 19–20 pathway it identified and has no qualifying parent, pregnancy, disability, or other categorical status, so income below 100% FPL does not confer eligibility." -us,scenario_090,head_medicaid_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model applied an unspecified income threshold without first establishing a Medicaid eligibility category. Kansas does not cover this 22-year-old childless adult through ACA expansion, and MAGI at 0.44 FPL is relevant only after a qualifying MAGI category exists." -us,scenario_090,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly invoked an adult expansion or generic low-income pathway in Kansas. Kansas has no ACA expansion-adult category, and this 22-year-old nondependent childless adult has no other categorical basis for Medicaid." -us,scenario_090,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Kansas expanded Medicaid and therefore applied the 138% FPL expansion threshold. Kansas has no such pathway, and the head's $20,300 in assets does not change the decisive failure to qualify under any Medicaid category." -us,scenario_090,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly named WIC's categorical requirement but then assumed categorical eligibility without any listed pregnancy, postpartum, breastfeeding, infant, or child-under-5 status. It treated the low income test as sufficient after that unsupported assumption, so it submitted eligible even though a 22-year-old adult outside the WIC categories is ineligible." -us,scenario_090,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model identified the correct components, $434.00 of Social Security tax and $101.50 of Medicare tax, and correctly excluded Additional Medicare Tax. It then made an arithmetic/transcription error by submitting $539.50 instead of the component sum of $535.50." -us,scenario_090,payroll_tax,minimax-m3,llm_error,other,False,"The model computed the correct FICA components and explicitly stated the correct total of $535.50, with no Kansas employee payroll tax and no Additional Medicare Tax. It then discarded that result and submitted $511.00 under an unsupported rounding step, even though standard payroll tax arithmetic does not round $535.50 to $511.00." -us,scenario_090,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the federal 130% FPL gross-income screen and standard SNAP benefit formula as sufficient for eligibility, then calculated an allotment from deductions and a maximum benefit. It missed PolicyEngine's Kansas SNAP eligibility threshold for this household, under which the roughly $7,020 annual income makes the single adult ineligible and yields $0." -us,scenario_090,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly excluded the $4,800 financial assistance from the income it used, but then applied a generic 130% FPL gross-income screen and BBCE asset treatment to keep the household eligible. It missed that PolicyEngine's Kansas SNAP eligibility threshold for this single-person household is exceeded by the roughly $7,020 annual income, so no allotment is payable." -us,scenario_090,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model spent its reasoning on whether financial assistance was countable and whether Kansas BBCE waived the asset test, then proceeded to the federal net-income allotment formula. It missed the controlling PolicyEngine Kansas SNAP eligibility threshold, which makes the household ineligible at roughly $7,020 annual income before any positive benefit is awarded." -us,scenario_090,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model acknowledged the resource test but overrode it through Kansas BBCE and then estimated a positive benefit from generic deduction and maximum-allotment parameters. It did not apply PolicyEngine's Kansas income eligibility threshold for this one-person household, which the roughly $7,020 annual income exceeds." -us,scenario_090,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model inferred positive SNAP from low cash income without performing the eligibility screen used by PolicyEngine for Kansas. The correct PolicyEngine derivation treats the roughly $7,020 annual income as above the applicable single-person SNAP eligibility threshold, so the shortcut to a low positive allotment is wrong." -us,scenario_090,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model applied annualized federal SNAP deductions and the FY2025 maximum allotment, then used Kansas BBCE to ignore assets and compute a positive benefit. It missed that PolicyEngine's Kansas SNAP eligibility threshold for this household is exceeded by the roughly $7,020 annual income, which makes the benefit $0." -us,scenario_090,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used estimated $3,500 and $2,250 deduction and exemption amounts instead of the applicable 2026 Kansas standard-deduction rules, creating $1,270 of taxable income when none remains. Its subsequent 5.2% calculation therefore taxes a nonexistent base." -us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted only a stale $3,500 standard deduction and omitted the applicable Kansas exemption and 2026 deduction treatment. That shortcut created $3,520 of taxable income, whereas the Kansas calculation reduces taxable income to zero." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied stale $3,500 and $2,320 deduction and exemption figures and incorrectly declared that a $7,000 low-income threshold controlled the result. Under the applicable 2026 Kansas deduction rules, the full $7,020 is below the taxable-income threshold, so there is no $1,200 tax base to which 5.2% applies." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used unsupported $3,600 and $2,320 deduction and exemption amounts, leaving a fabricated $1,100 of Kansas taxable income. The applicable 2026 standard-deduction rules eliminate the taxable base entirely." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model substituted stale $3,605 and $2,250 deduction and exemption figures, producing $1,165 of taxable income instead of zero. It then compounded that error by applying an obsolete 3.1% bottom rate." -us,scenario_090,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model deducted only $3,500 and omitted the rest of the applicable Kansas deduction and exemption treatment, leaving $3,520 as a taxable base. The 2026 Kansas calculation instead reduces taxable income to zero, so the 3.1% computation never applies." -us,scenario_090,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model asserted $1,120 of taxable income from unspecified estimated deduction and exemption figures. Applying the 2026 Kansas standard-deduction rules leaves no taxable income, so its 5.2% tax calculation starts from the wrong base." -us,scenario_090,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used stale $3,605 and $2,250 deduction and exemption amounts, incorrectly leaving $1,165 taxable. The applicable 2026 Kansas standard-deduction rules reduce taxable income to zero, making both its 5.2% rate application and rounding irrelevant." -us,scenario_090,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model subtracted only a stale $3,500 standard deduction, omitted the applicable Kansas exemption and 2026 deduction treatment, and thereby created $3,520 of taxable income. It also applied an inapplicable 5.7% flat rate to that nonexistent tax base." -us,scenario_090,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model invented a 2026 Kansas refundable EITC equal to 18% of the federal EITC. No applicable Kansas refundable EITC enters ks_refundable_credits for this household, so the computed state refundable credits are $0." -us,scenario_090,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly applied a Kansas refundable EITC equal to 17% of the federal EITC. That credit is not applicable under the 2026 Kansas rules used for this household, leaving ks_refundable_credits at $0." -us,scenario_090,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated 17% of the federal EITC as a Kansas refundable credit. Kansas supplies no applicable refundable EITC here, so rounding a federal EITC calculation to produce $91 is the wrong eligibility pathway." -us,scenario_090,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model treated low Kansas AGI as sufficient for the food sales tax credit and ignored its categorical eligibility requirement. A 22-year-old filer with no dependents and no disability does not qualify through the dependent-child, age, or disability categories, so the $125 credit is unavailable." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $1,170 non-Schedule-D capital-gains input in AGI and omitted the $627.91 overtime deduction. It also failed to reconcile its stated $1,425 calculation with its submitted $1,276; the traced tentative tax is $1,470 and the saver's credit reduces it to $1,350." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used the wrong standard deduction, omitted the traditional 401(k) exclusion and overtime deduction, included the $1,170 non-Schedule-D gain in income, and taxed qualified dividends at 15% instead of 0%. It then invented a full AOTC despite zero listed qualified education expenses." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model included the $1,170 non-Schedule-D gain in AGI, omitted the $627.91 overtime deduction, and failed to apply the $120 saver's credit. Those errors left it at tentative tax rather than reducing the traced $1,470 tentative tax to $1,350." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model treated the stated $27,000 gross wages as already net of the traditional 401(k), included the $1,170 non-Schedule-D gain, and omitted the overtime deduction. It also fabricated a $331 AOTC from zero qualified education expenses instead of applying the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model's own preferential-rate calculation produced about $1,684, but it submitted $2,109 without a supporting computation. It also included the $1,170 non-Schedule-D gain, omitted the overtime deduction, and failed to subtract the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction and obsolete bracket thresholds instead of the traced 2026 $16,100 deduction and applicable brackets. It also included the $1,170 non-Schedule-D gain, omitted the overtime deduction, and failed to apply the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model invented an AOTC despite zero listed qualified education expenses and then submitted $1,470 even though its own reasoning said that credit would leave $236. The correct nonrefundable reduction is the $120 saver's credit, which lowers tentative tax of $1,470 to $1,350." -us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the $1,800 other-health-insurance premium from wages, used a $15,000 standard deduction, included the $1,170 non-Schedule-D gain, and omitted the overtime deduction. Its near match is accidental; the traced calculation reaches $1,350 from $1,470 of tentative tax less the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model improperly deducted the $6,589 employer-sponsored insurance amount from the separately stated gross wages and used an unsupported deduction regime. It then applied an AOTC with zero qualified education expenses, incorrectly eliminating the liability." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $1,399 rests on an unsupported taxable-income estimate of $25,500. The trace instead produces $18,284.66 after the $16,100 standard deduction and $627.91 overtime deduction, then yields $1,350 after the saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $6,589 of employer-sponsored insurance from stated gross wages and applied a pre-TCJA personal-exemption framework. The traced 2026 calculation uses the $16,100 standard deduction and overtime deduction, producing $1,470 before the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $6,589 of employer-sponsored insurance from gross wages and used an $8,300 standard deduction plus a $5,300 personal exemption. It also treated the $1,170 non-Schedule-D gain as preferential income and omitted both the overtime deduction and saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $6,589 employer-sponsored insurance amount from stated gross wages and used a personal exemption rather than the traced $16,100 standard deduction. It also omitted the $627.91 overtime deduction and $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $15,400 standard deduction, included the $1,170 non-Schedule-D gain in AGI, and omitted the $627.91 overtime deduction. Although it applied the saver's credit, those taxable-income errors raised its result above $1,350." -us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model incorrectly applied an American Opportunity Credit despite zero listed qualified education expenses. The standard and overtime deductions do not eliminate taxable income; tentative tax is $1,470 and only the $120 saver's credit reduces it." -us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model used an unavailable American Opportunity Credit to erase the tax liability even though qualified education expenses are zero. The applicable nonrefundable credit is the $120 saver's credit, leaving $1,350." -us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model correctly identified the standard and overtime deductions but included the $1,170 non-Schedule-D gain in AGI and stated that no nonrefundable credit was usable. The traced tentative tax of $1,470 must be reduced by the $120 saver's credit to $1,350." -us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model applied the nonrefundable American Opportunity Credit even though no qualified education expenses were listed. With no AOTC, the $1,470 tentative tax is reduced only by the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model recognized the overtime deduction and correctly rejected the AOTC, but its $1,610 answer omitted the $120 saver's credit and did not reproduce the traced taxable-income computation. The applicable calculation yields $1,470 before that credit and $1,350 afterward." -us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly treated the AOTC and other nonrefundable credits as sufficient to eliminate liability. Qualified education expenses are zero, so no AOTC applies; the only traced nonrefundable reduction is the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied restored post-TCJA 10% and 15% brackets and a personal exemption instead of the operative 2026 $16,100 standard deduction and traced rate schedule. It also included the $1,170 non-Schedule-D gain and omitted the overtime deduction, overstating tentative tax before the saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $6,589 of employer-sponsored insurance from stated gross wages and used a personal exemption with an $8,350 standard deduction. It also included the $1,170 non-Schedule-D gain, omitted the overtime deduction, and failed to apply the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_091,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model used the American Opportunity Credit to eliminate liability even though qualified education expenses are zero. The refundable percentage of AOTC does not create eligibility for the credit, and the traced amount after the $120 saver's credit is $1,350." -us,scenario_091,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model double-counted the $628 overtime premium on top of annual wages that already include overtime and invented qualified education expenses for an AOTC. It also included the $1,170 non-Schedule-D gain and omitted the traditional 401(k), IRA, and overtime deductions required by the trace." -us,scenario_091,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model correctly derived zero qualified education expenses and zero refundable AOTC, then contradicted that derivation by inserting a $400 “minimal refundable AOTC assumption.” No minimum AOTC exists: with $0 of listed qualified education expenses, the refundable amount is $0." -us,scenario_091,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated student-status and income conditions as sufficient for an AOTC despite the prompt setting unlisted qualified education expenses to $0. It also misstated the refundable formula and submitted $1,200 even though the refundable AOTC is capped at $1,000; with no qualified expenses, it is $0." -us,scenario_091,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model awarded the maximum AOTC from enrollment, credential, 1098-T, and EIN facts without applying the qualified-education-expense base. The unlisted expense input is $0, producing a $0 AOTC and therefore a $0 refundable portion." -us,scenario_091,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model assumed a full $2,500 AOTC after checking personal eligibility but never established any qualified education expenses. Because unlisted expenses equal $0, there is no AOTC to split into nonrefundable and 40% refundable portions." -us,scenario_091,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model converted the maximum statutory AOTC into a $1,000 refundable credit solely from student eligibility facts. Qualified education expenses are the credit base, and the prompt supplies none, so the base and refundable credit are both $0." -us,scenario_091,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model awarded 40% of the maximum AOTC without computing the credit from qualified education expenses. Those expenses default to $0 under the prompt, so the eligible American Opportunity Credit and its refundable component are $0." -us,scenario_091,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for federal_refundable_credits, violating the required output contract." -us,scenario_091,federal_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model applied the 40% refundable share to the $2,500 maximum without first calculating an AOTC from qualified education expenses. With no listed qualified expenses, the AOTC is $0 and 40% of it is $0." -us,scenario_091,federal_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly eliminated the EITC based on the childless-worker income phaseout but awarded the maximum refundable AOTC merely from enrollment and documentation conditions. The prompt lists no qualified education expenses, so the AOTC calculation starts from a $0 expense base and yields no refundable credit." -us,scenario_091,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model invented a $528.18 Wisconsin employee payroll tax by treating state income-tax withholding as a payroll-tax component. Wisconsin income tax belongs in the separate state income-tax output, leaving only $1,674.00 of Social Security and $391.50 of Medicare tax." -us,scenario_091,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $1,800 other-health-insurance premium from FICA wages even though that listed expense does not reduce the payroll-tax base. Applying 7.65% to the full $27,000 produces $2,065.50." -us,scenario_091,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the $6,589 employer-sponsored-insurance-premium amount from FICA wages without a basis for treating it as an employee pre-tax payroll deduction. Social Security and Medicare apply to the full $27,000 wage base." -us,scenario_091,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly classified the $6,589 employer-sponsored-insurance-premium field as a pre-tax employee deduction from FICA wages. The applicable wage base remains $27,000, yielding $2,065.50." -us,scenario_091,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model correctly kept traditional 401(k) contributions in the FICA base but incorrectly removed the $6,589 insurance-premium amount. With no qualifying reduction to FICA wages, 7.65% applies to all $27,000." -us,scenario_091,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model reduced Social Security and Medicare wages to $20,411 by treating the $6,589 insurance-premium field as a pre-tax payroll exclusion. The trace uses $27,000 for both taxes, producing $1,674.00 and $391.50." -us,scenario_091,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct $27,000 wage base and the correct 6.2% and 1.45% rates but submitted an arithmetically inconsistent total. Those inputs equal $1,674.00 plus $391.50, or $2,065.50, not $2,063.10." -us,scenario_091,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's stated components already imply approximately $2,066, but it replaced their sum with an unsupported $2,118 estimate. Exact Medicare tax is $391.50, so the exact total is $2,065.50." -us,scenario_091,payroll_tax,gpt-5.6-terra,llm_error,other,False,"The model stated the correct component amounts but failed to add them correctly. $1,674.00 plus $391.50 equals $2,065.50, not $2,457.00." -us,scenario_091,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $6,589 employer-sponsored-insurance-premium amount from FICA wages. The applicable base is the full $27,000, and exact employee FICA is $2,065.50." -us,scenario_091,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required output contract." -us,scenario_091,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model added the $628 FLSA overtime premium to the $27,000 gross-wage total even though the prompt states that annual gross wages already include overtime pay. This double counting inflated the FICA base from $27,000 to $27,628." +us,scenario_090,federal_refundable_credits,claude-opus-4.8,llm_error,age_disability,False,"The model treated the 22-year-old as eligible for the childless EITC, missing its minimum age of 25. Its subsequent phase-in and phaseout estimate is irrelevant because age disqualifies the filer before the credit schedule is applied." +us,scenario_090,federal_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"The model applied the childless EITC schedule to a 22-year-old even though the minimum qualifying age is 25. It also submitted $721 after calculating a $535.50 phase-in amount, but the controlling error is that no EITC schedule applies to this filer." +us,scenario_090,federal_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model multiplied earnings by the 7.65% childless EITC phase-in rate without first applying the minimum-age test. At age 22, the filer is ineligible for the childless EITC, so this component is $0." +us,scenario_090,federal_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"The model expressly awarded the childless EITC to a single 22-year-old, overlooking the minimum qualifying age of 25. The 7.65% earnings calculation therefore produces no allowable credit." +us,scenario_090,federal_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"The model incorrectly declared that the 22-year-old qualifies for the childless EITC. Because childless eligibility begins at age 25, neither the phase-in rate nor the phaseout threshold is reached in the computation." +us,scenario_090,federal_refundable_credits,grok-4.3,llm_error,age_disability,False,"The model labeled the amount a full no-child EITC without applying the childless filer's minimum-age requirement. A 22-year-old cannot claim that credit, so the inferred $600 award is excluded." +us,scenario_090,federal_refundable_credits,grok-4.5,llm_error,age_disability,False,The model applied the 7.65% childless EITC phase-in rate directly to wages and checked only the income phaseout. It omitted the prior age test: the 22-year-old is below the minimum age of 25 and receives no EITC. +us,scenario_090,federal_refundable_credits,minimax-m3,llm_error,age_disability,False,"The model evaluated earnings, phase-in and phaseout ranges, and the investment-income limit but failed to enforce the childless EITC minimum age of 25. The head's age of 22 ends eligibility before any of those monetary tests can generate a credit." +us,scenario_090,federal_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,The model calculated a childless EITC from earnings while overlooking that the filer is only 22 and must be at least 25. Its unexplained reduction from $535.50 to $492 does not cure the categorical age disqualification. +us,scenario_090,federal_refundable_credits,qwen3.8-max,llm_error,age_disability,False,"The model attempted to place the filer on the childless EITC plateau and phaseout schedule without applying the minimum age of 25. At age 22 the filer receives no childless EITC, and the asserted $270 total has no valid refundable-credit component." +us,scenario_090,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model falsely stated that Kansas adopted Medicaid expansion and therefore applied the 138% FPL expansion-adult test. Kansas provides no such pathway, and the head's low income and assets do not overcome the absence of a qualifying Medicaid category." +us,scenario_090,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented an adult Medicaid category for a low-income person under age 26. Age 22 does not create a Kansas Medicaid pathway, and this nondependent head qualifies through none of the covered categories." +us,scenario_090,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated income below 100% FPL as sufficient for Medicaid eligibility despite recognizing that Kansas has not expanded Medicaid. In a non-expansion state, very low income alone does not cover a childless 22-year-old adult who belongs to no qualifying category." +us,scenario_090,head_medicaid_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model applied an unspecified income threshold without first establishing a covered Medicaid category. At age 22, the head is not eligible merely because MAGI equals about 0.44 FPL; the engine assigns the person to no Medicaid category." +us,scenario_090,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly invoked adult expansion or generic low-income rules in Kansas. Kansas has no expansion-adult pathway, and low income does not establish Medicaid eligibility for this childless 22-year-old outside every covered category." +us,scenario_090,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model falsely concluded that Kansas expanded Medicaid and applied the 138% FPL expansion threshold. The asset discussion is irrelevant to the decisive failure: this head has no qualifying Medicaid category, so income below 138% FPL cannot establish eligibility." +us,scenario_090,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model explicitly assumed categorical eligibility even though the prompt sets every unlisted status to false. At age 22, the head is neither an infant nor a child under five, and no pregnant, postpartum, or breastfeeding status is listed, so the mandatory WIC categorical test fails." +us,scenario_090,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model calculated both FICA components correctly but added $434.00 and $101.50 incorrectly: their sum is $535.50, not $539.50." +us,scenario_090,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly derived $434.00 of Social Security tax and $101.50 of Medicare tax, totaling $535.50, then replaced that result with an unsupported $511.00 final value. No rounding rule changes $535.50 to $511.00." +us,scenario_090,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model substituted the generic 130%-of-FPL gross-income screen for PolicyEngine's applicable Kansas eligibility threshold and therefore calculated an allotment for an ineligible household. Its final $2,137 also contradicts its own intermediate estimates of either $101 or $178 per month." +us,scenario_090,snap,claude-opus-4.8,llm_error,thresholds_rates,False,The model treated $585 per month as passing a generic 130%-of-FPL test and immediately applied the earned-income and standard deductions. PolicyEngine's Kansas eligibility threshold excludes this one-person household before those benefit-calculation deductions can produce an allotment. +us,scenario_090,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied generic 130%-of-FPL gross and 100%-of-FPL net tests, then calculated $93 per month after deductions. It missed the controlling Kansas eligibility threshold under which the household's approximately $7,020 of annual income makes SNAP $0; its extended speculation about educational assistance and BBCE does not alter that failed eligibility step." +us,scenario_090,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used the generic federal gross-income screen instead of the applicable Kansas eligibility threshold and therefore treated the household as entitled to a positive allotment. Its submitted $2,650 is also unsupported by its own calculation of about $99 per month, or $1,188 annually." +us,scenario_090,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,The model inferred eligibility from the household's seemingly low cash income without applying the Kansas SNAP eligibility threshold. PolicyEngine's eligibility calculation yields $0 before any positive allotment can be computed. +us,scenario_090,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed the household passed SNAP eligibility, then mechanically subtracted 30% of deducted net income from the maximum allotment using FY2025 parameters. It missed the prior Kansas threshold test that makes the household ineligible and prevents any allotment calculation." +us,scenario_090,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model explicitly rejected the applicable Kansas low-income exclusion and taxed $1,270 at an estimated 5.2% rate. It therefore bypassed the low-income threshold that reduces this household's Kansas liability to zero." +us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted only a $3,500 standard deduction, treated the remaining $3,520 as taxable, and applied 3.1%. It omitted the Kansas low-income treatment that leaves this $7,020-income household with no state tax liability." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model declared that the low-income exclusion stopped at $7,000 and taxed $1,200 at 5.2%. That threshold determination was wrong for the applicable Kansas calculation, under which the household's approximately $7,020 income produces zero liability." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model mechanically applied a 5.2% rate to $1,100 after estimated deductions and an exemption. It failed to apply the Kansas low-income threshold that eliminates tax at this household's income level." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied the obsolete 3.1% bottom rate to $1,165 of constructed taxable income. More fundamentally, it omitted the Kansas low-income threshold that makes the liability zero before any refundable credits." +us,scenario_090,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model deducted only $3,500 from Kansas AGI and taxed the resulting $3,520 at 3.1%. It omitted the applicable Kansas low-income treatment, so it created taxable liability where the state calculation yields zero." +us,scenario_090,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied 5.2% to an estimated $1,120 taxable-income base and assumed no further reduction. It missed the Kansas low-income threshold that eliminates this household's state income tax." +us,scenario_090,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model taxed $1,165 at 5.2% after subtracting an estimated standard deduction and personal exemption. It failed to apply the Kansas low-income threshold that reduces the liability to zero." +us,scenario_090,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model subtracted only the standard deduction, omitted the remaining low-income relief, and then applied an incorrect 5.7% flat rate to $3,520. The applicable Kansas calculation produces no tax at approximately $7,020 of income." +us,scenario_090,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used approximate deductions, an approximate personal exemption, and an approximate 5.5% rate to construct a positive tax. It omitted the Kansas low-income threshold that makes this household's liability zero." +us,scenario_090,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model applied the federal EITC phase-in rate without checking the childless claimant age rule. At age 22 with no qualifying child, the head receives no federal EITC, so applying Kansas’s percentage to a fabricated $535.50 federal credit incorrectly produced $96.39." +us,scenario_090,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model treated low earned income as sufficient for the federal childless EITC and then applied a Kansas percentage. The 22-year-old head is below the minimum age for a claimant without a qualifying child, leaving no federal EITC on which to calculate a Kansas credit." +us,scenario_090,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model calculated a childless federal EITC of $536 but omitted the federal minimum-age condition. Because the head is 22 and has no qualifying child, the federal EITC is zero and the derivative Kansas credit is also zero." +us,scenario_090,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model treated the Kansas Food Sales Tax Credit’s income ceiling as its sole eligibility test. A claimant must also be age 55 or older, disabled or blind, or have a dependent child under 18; this 22-year-old single filer meets none of those categories, so the $125 credit is unavailable." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model included the $1,170 non-Schedule-D capital-gain input in federal AGI and omitted the $627.91 overtime deduction, producing the wrong taxable-income base. It also submitted $1,276 despite its final recomputation yielding $1,425 instead of the required $1,470 tentative tax less the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model invented a full $2,500 American Opportunity Credit even though qualified education expenses were unlisted and therefore zero. It also used the wrong standard deduction and taxed qualified dividends at 15% instead of applying the 0% qualified-dividend band." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model included the $1,170 non-Schedule-D capital-gain input in AGI and omitted the $627.91 overtime deduction. It then failed to subtract the $120 saver's credit from the resulting tentative tax." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model treated the stated $27,000 gross wages as already net of the traditional 401(k), included the $1,170 capital-gain input in AGI, and omitted the overtime deduction. It also invented an unsupported $331 AOTC amount despite zero listed qualified education expenses." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Although its intermediate ordinary-income computation was near $1,684, the model abandoned that calculation and taxed the full $20,083 base to reach $2,109. It also included the $1,170 capital-gain input, omitted the overtime deduction, and failed to apply the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction instead of $16,100 and omitted the $627.91 overtime deduction. It also included the $1,170 capital-gain input in AGI and applied no $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's stated calculation would leave roughly $236 after its invented AOTC, yet it submitted $1,470, so the submitted value does not follow from its reasoning. The correct $1,470 is tentative tax before the $120 saver's credit, making the requested result $1,350." +us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the separately listed $1,800 health premium from wages, used a $15,000 standard deduction, included $1,170 of capital gains, and omitted the $627.91 overtime deduction. Those offsetting errors brought it close by coincidence; the exact tentative tax is $1,470 and the exact saver's credit is $120, yielding $1,350 rather than $1,351." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model improperly deducted the $6,589 employer-sponsored insurance premium from stated gross wages and invented both a personal exemption and an AOTC despite zero listed qualified education expenses. The standard and overtime deductions instead leave $1,470 of tentative tax, of which only the $120 saver's credit is subtracted." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted explanation used taxable income of about $25,500, whereas the standard deduction and overtime deduction produce $18,284.66 before separating qualified dividends. The resulting ordinary-rate computation is $1,470, followed by the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $6,589 employer-sponsored insurance premium from stated wages and applied a pre-TCJA personal exemption. Those deductions understated the ordinary taxable base; the applicable deductions produce $1,470 of tentative tax before the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $6,589 of employer-sponsored insurance from gross wages and used a post-sunset $8,300 standard deduction plus $5,300 personal exemption. PolicyEngine instead applies the $16,100 standard deduction and $627.91 overtime deduction, then subtracts the $120 saver's credit from $1,470." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $6,589 employer-sponsored insurance premium from stated gross wages and invoked a personal exemption. It also omitted the $120 saver's credit that reduces the correctly computed $1,470 tentative tax to $1,350." +us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $15,400 standard deduction, included the $1,170 capital-gain input in AGI, and omitted the $627.91 overtime deduction. It also recalculated saver's-credit AGI by subtracting the 401(k) and IRA deductions a second time; the engine calculation yields exactly $1,470 less $120." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model asserted that the standard deduction and AOTC eliminated the liability, but the listed facts contain no qualified education expenses and therefore generate no AOTC. Taxable income remains $18,284.66, producing $1,470 before the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model used an American Opportunity Credit without any listed qualified education expenses. The available nonrefundable credit is the $120 saver's credit, which reduces $1,470 of tentative tax to $1,350 rather than zero." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used AGI of $36,183 because it included the $1,170 non-Schedule-D capital-gain input, rather than the traced AGI of $35,012.56. It also stopped at the $1,470 tentative tax and failed to subtract the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model invented a usable nonrefundable American Opportunity Credit even though qualified education expenses were unlisted and therefore zero. Only the $120 saver's credit offsets the $1,470 tentative tax." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model recognized the standard deduction, overtime deduction, and 0% qualified-dividend rate but miscomputed the remaining ordinary-rate tax as $1,610 rather than $1,470. It also omitted the $120 saver's credit from the requested after-nonrefundable-credit output." +us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model treated the AOTC as available without any listed qualified education expenses and consequently reduced the liability to zero. The listed retirement contributions instead generate a $120 saver's credit against $1,470 of tentative tax." +us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied restored post-TCJA 10% and 15% brackets and a personal exemption rather than the applicable 2026 $16,100 standard deduction and 10%/12% brackets. It also included the $1,170 capital-gain input in AGI and omitted the overtime deduction." +us,scenario_091,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $6,589 employer-sponsored insurance premium from stated gross wages and used a personal exemption with an $8,350 standard deduction. It also omitted the $627.91 overtime deduction and the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset personal exemption, an $8,300 standard deduction, and a restored 15% bracket. The applicable calculation uses a $16,100 standard deduction, the $627.91 overtime deduction, 10%/12% brackets, and an exact $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not be evaluated as a substantive computation." +us,scenario_091,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model applied an American Opportunity Credit despite zero listed qualified education expenses. The refundable character of part of a potential AOTC does not create the credit; the actual pre-refundable result is $1,470 less the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model double-counted the $628 overtime premium on top of annual gross wages even though the prompt says wages already include overtime, then invented qualified education expenses and an AOTC. With no listed education expenses, the AOTC is zero and the only applied nonrefundable credit is the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that a single-filer standard deduction reduced more than $40,000 of stated taxable income to zero, which is arithmetically false. The traced deductions leave $18,284.66 of taxable income and $1,470 of tentative tax before the $120 saver's credit." +us,scenario_091,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived a $0 refundable AOTC from $0 qualified education expenses, then discarded that derivation and inserted a $400 “minimal refundable AOTC assumption.” The AOTC has no minimum refundable amount, so $0 of expenses produces $0 of refundable credit." +us,scenario_091,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model treated enrollment, credential, 1098-T, and institution-EIN facts as sufficient to generate an AOTC despite the prompt setting unlisted qualified education expenses to $0. It also misstated the refundable AOTC calculation: the refundable portion is 40% of the expense-based AOTC, capped at $1,000, not $1,200 or 15% of education expenses." +us,scenario_091,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model awarded the maximum $1,000 refundable AOTC solely from student-status and documentation conditions. Those conditions establish only potential eligibility; with no listed qualified education expenses, the AOTC base is $0 and its refundable portion is $0." +us,scenario_091,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model assumed a full AOTC without any qualified education expenses. Tax liability and the under-24 refundable-credit restriction do not create an expense-based credit: the prompt assigns $0 to unlisted tuition expenses, yielding a $0 AOTC before either issue matters." +us,scenario_091,federal_refundable_credits,gemini-3-flash-preview,llm_error,other,False,"The model converted the AOTC’s $1,000 refundable maximum into an automatic award based on student eligibility. With $0 qualified education expenses, the underlying AOTC is $0 and 40% of it remains $0." +us,scenario_091,federal_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The model applied 40% to the $2,500 maximum AOTC instead of calculating the credit from actual qualified education expenses. The expense input is unlisted and therefore $0, so no refundable AOTC arises." +us,scenario_091,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied neither the requested numeric output nor an explanation. It therefore failed the required output contract rather than making an assessable substantive calculation. +us,scenario_091,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model treated 40% of the maximum AOTC as the household’s refundable credit without first computing an expense-based AOTC. Since qualified education expenses are unlisted and equal $0, the refundable portion is $0 rather than the $1,000 cap." +us,scenario_091,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model correctly eliminated the childless EITC based on income but awarded the maximum refundable AOTC from enrollment and documentation facts alone. No qualified education expenses were listed, so the AOTC calculation starts at $0 and produces no refundable portion; nonrefundable credits offsetting tax liability does not change that." +us,scenario_091,head_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,The model incorrectly placed MAGI of 2.19 times FPL below Wisconsin's Medicaid income threshold for a single adult. It also treated that asserted income threshold as sufficient despite the head qualifying through no Medicaid eligibility category. +us,scenario_091,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly added $528.18 of estimated Wisconsin income-tax withholding to payroll tax. Wisconsin income tax belongs in the separate state income-tax output and is not a mandatory employee state payroll tax, leaving only $1,674.00 of Social Security and $391.50 of Medicare tax." +us,scenario_091,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly deducted the separately listed $1,800 of other health-insurance premiums from FICA wages. Applying 6.2% Social Security and 1.45% Medicare to the full $27,000 wage base yields $2,065.50." +us,scenario_091,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the $6,589 employer-sponsored-insurance premium as an employee pre-tax payroll deduction and subtracted it from FICA wages. The payroll-tax base remains the full $27,000 of annual wages, producing $2,065.50." +us,scenario_091,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted the $6,589 employer-sponsored-insurance premium from wages without a fact establishing an employee pre-tax payroll deduction. Social Security and Medicare apply to the full $27,000 wage base, not $20,411." +us,scenario_091,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model correctly kept traditional 401(k) contributions in the FICA base but incorrectly removed the $6,589 employer-sponsored-insurance premium. The applicable FICA wage base is $27,000, yielding $2,065.50 at the combined 7.65% rate." +us,scenario_091,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model incorrectly reduced Social Security and Medicare wages to $20,411 by subtracting the $6,589 employer-sponsored-insurance premium. The full $27,000 is subject to both employee FICA components." +us,scenario_091,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct bases and rates but submitted an arithmetically inconsistent total. Its own components are $1,674.00 of Social Security plus $391.50 of Medicare, which sum to $2,065.50 rather than $2,063.10." +us,scenario_091,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model derived approximately the correct FICA components and then replaced their sum with an unsupported $2,118.00 'annual payroll-tax estimation.' Exact calculation gives $1,674.00 + $391.50 = $2,065.50, with no further rounding adjustment." +us,scenario_091,payroll_tax,gpt-5.6-terra,llm_error,other,False,"The model listed the correct Social Security and Medicare components but submitted a total that does not equal them. $1,674.00 plus $391.50 is $2,065.50, not $2,457.00." +us,scenario_091,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly treated the $6,589 employer-sponsored-insurance premium as a pre-tax reduction of FICA wages. Using the full $27,000 wage base produces $2,065.50 rather than the rounded $1,561." +us,scenario_091,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required structured-output contract." +us,scenario_091,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model double-counted the $628 FLSA overtime premium by adding it to the $27,000 gross annual wage total. The prompt specifies that gross wages already include overtime pay, so FICA applies to $27,000 and equals $2,065.50." us,scenario_091,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used AGI near $36,183 and an estimated standard deduction near $10,668 instead of Wisconsin AGI of $35,012.56 and the $12,067.66 deduction. Its submitted $1,257 also contradicts its own refined tax calculation of roughly $975." -us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly applied the federal American Opportunity Tax Credit against Wisconsin income tax. It also started from federal taxable income rather than computing Wisconsin taxable income as $35,012.56 minus the $12,067.66 state standard deduction and $700 exemption." -us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model overstated Wisconsin AGI at about $36,183 and understated the sliding-scale standard deduction at about $9,500. The exact inputs produce $22,244.90 of Wisconsin taxable income, not roughly $25,983." -us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model understated the Wisconsin standard deduction at about $7,700 and used AGI of about $36,183, inflating taxable income to roughly $27,783. The correct state taxable income is $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used approximate AGI of $36,183 and a standard deduction near $9,000 instead of $35,012.56 and $12,067.66. That overstated the income exposed to Wisconsin's brackets by more than $4,000." -us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model substituted estimated and then 2024 Wisconsin parameters for the requested 2026 parameters. It also used $36,183 of AGI and a $9,688 deduction rather than the traced $35,012.56 AGI and $12,067.66 deduction." -us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model invented a Wisconsin nonrefundable Married/Single credit for this single filer and treated the personal exemption as a credit after already using it in taxable-income reasoning. It also used approximate income and deduction amounts instead of taxable income of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used 2025 rules, incorrect rates of 3.54% and 4.65%, and AGI of $34,383. The requested calculation uses the 2026 Wisconsin schedule and taxable income of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model omitted taxable income components and reduced Wisconsin AGI to $29,594. The traced AGI is $35,012.56, which after the exact deduction and exemption leaves $22,244.90 taxable." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer treated the $27,000 wage figure as the relevant Wisconsin AGI and did not incorporate the taxable investment income and state adjustments. The correct derivation starts from Wisconsin AGI of $35,012.56 and yields $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The answer names the general method but does not apply the household's full taxable income or the exact sliding-scale deduction. Wisconsin taxable income is $22,244.90, whose bracket tax is $843.66 rather than $588." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used Wisconsin AGI of $29,594 and a $12,707 deduction, omitting taxable income included in the state AGI. The correct inputs are $35,012.56 of AGI, a $12,067.66 deduction, and a $700 exemption." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model applied a 60% long-term capital-gain exclusion and an unspecified medical-expense credit. Those reductions do not belong in this traced liability; the applicable state computation leaves $22,244.90 taxable and $843.66 due." -us,scenario_091,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly phased the Wisconsin standard deduction down to zero. The household receives a $12,067.66 standard deduction, so only $22,244.90 is taxable rather than the model's full $36,183 estimate." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and state treatment eliminate the liability without calculating them. The standard deduction and exemption reduce $35,012.56 of AGI to $22,244.90, which still produces $843.66 of tax." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model used the American Opportunity Credit facts to eliminate Wisconsin tax, but that federal credit does not offset this Wisconsin liability. The state bracket tax on $22,244.90 remains $843.66 before refundable credits." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model made unsupported subtractions for overtime, health insurance, and capital gains and arrived at Wisconsin income of $33,404. The traced Wisconsin AGI is $35,012.56, and its exact standard deduction is $12,067.66." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used an unspecified estimate for Wisconsin taxable income and the standard deduction. Applying $35,012.56 of AGI, the $12,067.66 deduction, and the $700 exemption yields $843.66, not $943." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's rounded estimate did not carry the exact Wisconsin AGI, capital-gain treatment, deduction, and bracket inputs through the calculation. Those inputs produce $22,244.90 of taxable income and exactly $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model gave only a rounded estimate and omitted the exact Wisconsin deduction computation. The traced $12,067.66 standard deduction and $700 exemption reduce state AGI to $22,244.90 taxable, producing $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model incorrectly asserted that Wisconsin deductions and credits fully offset the tax. No stated nonrefundable credit erases the $843.66 bracket tax on $22,244.90 of taxable income." -us,scenario_091,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used federal AGI of $36,183 and derived taxable income near $24,437. Wisconsin AGI is $35,012.56 and the exact deduction plus exemption leaves only $22,244.90 taxable." -us,scenario_091,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated Wisconsin AGI at $29,594 and therefore taxed only $16,811. The complete state income calculation starts at $35,012.56 and yields $22,244.90 after the standard deduction and exemption." -us,scenario_091,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_091,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used Wisconsin income of $35,832 and a standard deduction of $11,868.47. The traced amounts are $35,012.56 and $12,067.66, leaving $22,244.90 taxable and producing $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model incorrectly treated the Wisconsin School Property Tax Credit as fully offsetting the liability despite no qualifying property-tax or rent input. The Wisconsin income-tax computation therefore retains the $843.66 liability. -us,scenario_091,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model invented a deduction for half of the employer-sponsored insurance premium, used a flat guessed standard deduction, and mischaracterized the $700 personal exemption as a credit equal to 11% of that amount. Wisconsin instead uses AGI of $35,012.56, a $12,067.66 sliding-scale deduction, and a direct $700 exemption from income." +us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used AGI near $36,183 and reduced the Wisconsin standard deduction to roughly $10,668 instead of using Wisconsin AGI of $35,012.56 and the $12,067.66 deduction. It also submitted $1,257 after its own bracket calculation yielded about $975, so its final value did not follow its stated computation." +us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly applied the federal American Opportunity Tax Credit against Wisconsin income tax. The federal AOTC does not eliminate this state liability; Wisconsin taxable income is $22,244.90 and its pre-refundable-credit tax is $843.66." +us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model overstated Wisconsin AGI at about $36,183 and understated the sliding standard deduction at about $9,500. The trace instead uses $35,012.56 of AGI and a $12,067.66 standard deduction, leaving $22,244.90 taxable rather than roughly $25,983." +us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model understated Wisconsin's sliding standard deduction as about $7,700 and consequently placed roughly $27,783 into the brackets. The applicable deduction is $12,067.66, and after the $700 exemption taxable income is $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used approximate AGI of $36,183 and a standard deduction near $9,000, overstating taxable income at roughly $26,500. Wisconsin AGI is $35,012.56 and the applicable standard deduction is $12,067.66, producing $22,244.90 of taxable income." +us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model substituted estimated 2024 parameters and inflation projections for the applicable 2026 Wisconsin parameters. It also began from $36,183 rather than $35,012.56, whereas the correct 2026 deduction is $12,067.66 and taxable income is $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model used $36,183 of income and an invented standard deduction of roughly $11,340 instead of the traced $35,012.56 AGI and $12,067.66 deduction. It also invoked a nonexistent applicable Married/Single credit for this single filer." +us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used 2025 rules, including 3.54% and 4.65% rates, rather than Wisconsin's applicable 2026 schedule. It also used $34,383 of AGI and a flat $13,230 deduction instead of $35,012.56 AGI and the income-dependent $12,067.66 deduction." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model omitted taxable income components when it set Wisconsin AGI to $29,594 rather than $35,012.56. After the correct $12,067.66 standard deduction and $700 exemption, taxable income is $22,244.90, not $17,484." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer treated the $27,000 wage amount as the relevant Wisconsin AGI and did not account for the traced wage adjustments and taxable investment income that produce $35,012.56. The correct deduction-and-exemption calculation leaves $22,244.90 taxable and yields $843.66." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The submitted $588 does not reflect the traced Wisconsin tax base: $35,012.56 AGI less the $12,067.66 standard deduction and $700 exemption. That computation leaves $22,244.90 subject to the 2026 brackets and produces $843.66." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated Wisconsin AGI at $29,594 and overstated the standard deduction at $12,707. The correct figures are $35,012.56 and $12,067.66, leaving $22,244.90 taxable rather than $16,887." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied a 60% long-term capital-gain exclusion and an allowable medical-expense credit to this fact pattern. The traced Wisconsin calculation instead reaches $35,012.56 of AGI and $22,244.90 of taxable income, with no such reductions eliminating the resulting $843.66 tax." +us,scenario_091,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated Wisconsin's standard deduction as fully phased out. The filer receives a $12,067.66 standard deduction, and after the $700 exemption only $22,244.90—not $36,183—is subject to the brackets." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and state treatment eliminate the liability without performing the Wisconsin taxable-income calculation. AGI of $35,012.56 less the $12,067.66 standard deduction and $700 exemption leaves $22,244.90 taxable, producing $843.66." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model incorrectly used the American Opportunity Tax Credit facts to eliminate Wisconsin income tax. That federal credit does not offset this Wisconsin liability, which is $843.66 on $22,244.90 of state taxable income." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model created unsupported subtractions for overtime, health-insurance premiums, and capital gains and thereby reduced Wisconsin income to $33,404. The traced Wisconsin AGI is $35,012.56, and the applicable $12,067.66 deduction plus $700 exemption yields $22,244.90 taxable." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used an approximate Wisconsin tax base rather than the income-dependent deduction calculation. The exact sequence is $35,012.56 of AGI less $12,067.66 and $700, leaving $22,244.90 taxable and $843.66 of tax." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied an unspecified capital-gain subtraction and approximated the deduction and rate calculation. The traced inputs produce $35,012.56 of AGI, a $12,067.66 standard deduction, a $700 exemption, and exactly $843.66 of tax." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model did not apply the exact 2026 Wisconsin deduction and bracket parameters. Using $35,012.56 of AGI, the $12,067.66 standard deduction, and the $700 exemption produces $22,244.90 taxable and $843.66, not the rounded $860 estimate." +us,scenario_091,state_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly asserted that Wisconsin deductions and credits fully offset the tax. The deductions leave $22,244.90 taxable, and no applicable nonrefundable credit reduces the resulting $843.66 liability to zero." +us,scenario_091,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model started from overstated AGI of $36,183 and used an approximate deduction that left about $24,437 taxable. The traced Wisconsin figures leave only $22,244.90 taxable after the $12,067.66 standard deduction and $700 exemption." +us,scenario_091,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model omitted taxable income components by setting Wisconsin AGI to $29,594 rather than $35,012.56. It also used an incorrect $12,783 deduction and omitted the $700 exemption from its displayed taxable-income calculation." +us,scenario_091,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used approximate AGI and deduction figures and applied a 4.65% marginal rate instead of the applicable 2026 Wisconsin schedule used in the trace. The exact taxable income is $22,244.90 and the resulting tax is $843.66." +us,scenario_091,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required output contract." +us,scenario_091,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used AGI of $35,832, an unsupported 30% exclusion for the non-Schedule-D gain, and an incorrect $11,868.47 standard deduction. The trace instead uses $35,012.56 of AGI and a $12,067.66 deduction, leaving $22,244.90 taxable." +us,scenario_091,state_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,The model incorrectly treated the Wisconsin School Property Tax Credit as fully offsetting the liability even though no qualifying property-tax or rent input was listed. The Wisconsin taxable-income calculation produces $843.66 before refundable credits. +us,scenario_091,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model improperly deducted half of the employer-sponsored insurance premium from wages, treated the $700 personal exemption as a $77 credit, and cycled through unsupported flat standard deductions and obsolete rates. Wisconsin instead uses $35,012.56 of AGI, a $12,067.66 standard deduction, and a $700 exemption deduction, yielding $843.66." +us,scenario_091,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model invented nonrefundable Wisconsin credits sufficient to erase $1,374.94 of calculated tax. It also used $40,571 of income and a flat $12,200 deduction instead of the traced $35,012.56 AGI, $12,067.66 standard deduction, and $700 exemption." us,scenario_091,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_092,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated total cash income of $32,380 as taxable gross income and included the full $17,400 Social Security benefit in taxable income. The correct Social Security computation uses provisional income of $23,680, below the $25,000 single-filer threshold, so none of the Social Security benefit is taxable and the remaining $14,980 pension is fully offset by the senior standard deduction." -us,scenario_092,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model applied the 85% Social Security inclusion rule even though its stated threshold test used the wrong measure: provisional income is $14,980 plus one-half of Social Security, not pension plus all Social Security. That provisional income is $23,680, below the $25,000 single threshold, so taxable Social Security is $0 and the $14,980 pension is eliminated by the standard deduction." -us,scenario_092,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model included $3,700 of Social Security in AGI and then itemized deductions against that inflated AGI. The correct provisional-income test makes taxable Social Security $0, leaving only $14,980 of AGI; the age-77 standard deduction is larger than that itemized amount and reduces taxable income to $0." -us,scenario_092,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly recognized that annual Social Security plus pension income of $32,380 exceeds the aged/disabled income threshold it cited, but then submitted the opposite eligibility value. It treated an income-failing aged/disabled pathway as eligible instead of carrying the failed income test through to medicaid_category = NONE." -us,scenario_092,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated being age 65+ and disabled as meeting Alabama's aged/disabled Medicaid category. PolicyEngine requires an actual qualifying pathway; with SSI received equal to 0 and countable income not qualifying under the SSI-related route, the head does not enter an aged/disabled Medicaid category." -us,scenario_092,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model used the shortcut that elderly and disabled status implies Medicaid eligibility. PolicyEngine does not make age or disability alone sufficient in Alabama; the head qualifies through no Medicaid category, and MAGI at 2.03 times FPL does not qualify under MAGI categories." -us,scenario_092,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model mislabeled $32,380 of annual Social Security and pension income as low enough for Alabama elderly/disabled Medicaid. PolicyEngine's pathway checks produce medicaid_category = NONE, so the age/disability facts do not overcome the failed income/category tests." -us,scenario_092,head_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model applied an approximately $2,829/month SSI-related income limit, which is the wrong standard for this Alabama aged/disabled Medicaid determination. Under PolicyEngine's Alabama pathways, the head's income and lack of SSI do not qualify them for an aged/disabled category, leaving medicaid_category = NONE." -us,scenario_092,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that payroll tax applies only to wage income, then contradicted that rule by treating the $17,400 Social Security retirement benefit as if it were Social Security- and Medicare-taxable wages. Applying 6.2% plus 1.45% to Social Security benefits produced $1,253.76, but retirement benefits and pension income are outside the employee payroll tax base when no wages are listed." +us,scenario_092,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the full $17,400 Social Security benefit as gross income instead of applying the provisional-income formula, under which none of it is taxable. With AGI of $14,980, the aged-single standard deduction eliminates all taxable income." +us,scenario_092,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included 85% of Social Security by comparing total income directly with the $34,000 threshold; Social Security taxation uses provisional income, including only one-half of benefits, and yields no taxable Social Security here. It also inconsistently applied the basic and age-based standard-deduction amounts, but the correct $14,980 AGI is fully offset by the applicable deduction." +us,scenario_092,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $3,700 of taxable Social Security and used an approximately $8,500 standard deduction, causing it to itemize $9,028 instead. No Social Security is taxable under the provisional-income calculation, and the aged-single standard deduction exceeds the resulting $14,980 AGI." +us,scenario_092,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model’s reasoning correctly derived zero taxable income after the standard deduction but submitted $1,613 anyway. That final value contradicts its own stated computation and does not follow from any tax calculation it presented." +us,scenario_092,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly reasoned that $32,380 of annual Social Security and pension income exceeds the aged/disabled income standard, but then submitted 1 instead of the 0 required by its own analysis. It also invoked medically needy eligibility without establishing that the person qualified through that pathway." +us,scenario_092,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated being age 65 or older and disabled as sufficient for Alabama's aged/disabled Medicaid category. Those traits only identify a potential pathway; the person receives no SSI and does not satisfy its financial eligibility requirements, leaving the Medicaid category as NONE." +us,scenario_092,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model converted age 77 and disability directly into Medicaid eligibility without applying any Alabama categorical and financial test. Neither characteristic creates automatic eligibility, and the person qualifies through no Medicaid pathway." +us,scenario_092,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model labeled $32,380 of annual unearned income as low enough for elderly/disabled Medicaid without applying Alabama's pathway-specific eligibility rules. The person receives no SSI and qualifies through no aged, disabled, MAGI, or other Medicaid category." +us,scenario_092,head_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model applied an approximately $2,829 monthly SSI-related income limit that does not establish eligibility for this Alabama case, then compared income after a $20 disregard to that incorrect standard. The person receives no SSI and qualifies through no aged/disabled category, so that threshold calculation cannot produce Medicaid eligibility." +us,scenario_092,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly recognized that the household has no wages, then improperly invented an “implied Social Security-taxable earnings equivalent” equal to the $17,400 Social Security retirement benefit. Retirement benefits are not employee payroll-taxable earnings, so applying the combined 7.65% FICA rate to that amount produced the erroneous $1,253.76 instead of $0." us,scenario_092,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model applied Alabama’s standard deduction directly to the pension without first excluding the $6,000 of retirement income available to a taxpayer age 65 or older. After that exclusion, the household’s itemized deductions eliminate all remaining Alabama income." -us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model correctly selected the $9,028 itemized deduction but treated all $14,980 of private pension income as Alabama income. The age-65 retirement-income exclusion reduces that amount by $6,000, leaving less income than the itemized deductions and therefore no taxable income." -us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model omitted the $6,000 Alabama retirement-income exclusion for taxpayers age 65 or older and used the standard deduction instead of the larger listed itemized deductions. Those two errors created $9,480 of taxable income where the correct computation has none." -us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model correctly exempted Social Security and totaled the itemized deductions, but it failed to subtract Alabama’s $6,000 age-65 retirement-income exclusion from the private pension. The remaining $8,980 is less than the $9,028 itemized deduction, producing zero taxable income." -us,scenario_092,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model substituted an age exemption and standard deduction for Alabama’s separate $6,000 exclusion of retirement income for taxpayers age 65 or older. It also failed to choose the $9,028 itemized deduction, which exceeds the remaining $8,980 of pension income." -us,scenario_092,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model’s bracket estimate left taxable pension income after generic deductions because it did not apply Alabama’s $6,000 age-65 retirement-income exclusion and the household’s $9,028 itemized deduction in sequence. Those adjustments reduce Alabama taxable income to zero before the rate schedule applies." -us,scenario_092,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model correctly exempted Social Security and selected the $9,028 itemized deduction, but it omitted Alabama’s $6,000 retirement-income exclusion for a taxpayer over 65. That exclusion lowers the pension amount to $8,980, which the itemized deduction fully offsets." -us,scenario_092,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model’s $182 result reflects taxing retirement income after ordinary deductions without applying Alabama’s $6,000 retirement-income exclusion for taxpayers age 65 or older. Once that exclusion and the $9,028 itemized deduction are applied, no taxable income remains." -us,scenario_092,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not submit a value or explanation for the requested output. -us,scenario_092,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model omitted Alabama’s $6,000 age-65 retirement-income exclusion and used a $2,500 standard deduction despite the household having $9,028 of itemized deductions. The exclusion leaves $8,980, which is entirely offset by those itemized deductions." -us,scenario_092,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model incorrectly included Social Security in Alabama income, producing federal AGI of $29,770, and also omitted Alabama’s $6,000 age-65 retirement-income exclusion from the private pension. Social Security is exempt, and the remaining pension income after the retirement exclusion is fully offset by the $9,028 itemized deduction." -us,scenario_092,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model invented age-based standard-deduction amounts and an adjustment that increased its initially computed $349 tax to $852.45, even though deducting federal income tax cannot increase Alabama taxable income. It also omitted the $6,000 age-65 retirement-income exclusion and the larger $9,028 itemized deduction, which together leave no taxable income." +us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model applied Alabama deductions and rates to retirement income without first excluding the qualifying private pension from Alabama taxable income. Once the pension and Social Security are excluded, no income remains to produce its $639 liability." +us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model correctly excluded Social Security but incorrectly retained the entire $14,980 qualifying private pension in Alabama AGI. Alabama excludes that pension, so its itemized-deduction and bracket calculation starts from zero taxable retirement income and yields $0." +us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model treated the $14,980 private pension as Alabama income and merely reduced it by the standard deduction and exemptions. The qualifying pension is excluded before those deductions, leaving no Alabama taxable income." +us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model excluded Social Security but failed to apply Alabama's exclusion for the qualifying private pension. Its $182.60 is the tax on pension income after itemized deductions and a personal exemption, whereas excluding the pension leaves $0 taxable income." +us,scenario_092,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model incorrectly kept the $14,980 qualifying private pension in Alabama AGI, then applied the standard deduction and personal and age exemptions. Alabama excludes the pension itself, so the bracket liability is $0 rather than $434." +us,scenario_092,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model treated the private pension as taxable Alabama income subject only to deductions and exemptions. Alabama's pension exclusion removes that income before the bracket calculation, producing no liability." +us,scenario_092,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model correctly removed Social Security but incorrectly defined Alabama AGI as the full $14,980 private pension. The qualifying pension is also excluded under Alabama law, so the subsequent itemized-deduction and rate calculation should never produce $182.60." +us,scenario_092,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model's $182 estimate reflects taxing the private pension after Alabama deductions instead of applying Alabama's pension-income exclusion. With both the pension and Social Security excluded, the liability before refundable credits is $0." +us,scenario_092,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model incorrectly retained the $14,980 qualifying private pension as Alabama AGI while excluding only Social Security. Excluding the pension as Alabama requires leaves no taxable income for its $182.60 bracket calculation." +us,scenario_092,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, violating the required output contract." +us,scenario_092,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model treated the qualifying private pension as Alabama taxable income and reduced it only by a standard deduction and personal exemption. Alabama excludes the pension at the income stage, leaving no amount to which the tax brackets apply." +us,scenario_092,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model constructed $29,770 of Alabama AGI by retaining the private pension and a taxable portion of Social Security, even though Alabama excludes both sources here. It then substituted exemptions for the required retirement-income exclusions, creating taxable income that does not exist." +us,scenario_092,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model incorrectly retained the qualifying private pension in Alabama taxable income and attempted to approximate deductions and a federal-tax deduction instead of applying the pension exclusion. Its final $852.45 also contradicts its own stated bracket calculation of $349; excluding the pension yields $0 before either calculation. us,scenario_092,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_093,dependent1_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated being disabled as sufficient for Missouri Medicaid through a disability-related pathway. PolicyEngine requires an actual qualifying eligibility category for that pathway, and this person has none: SSI received is 0, the person remains a tax-unit dependent, and MAGI is 2.82 times FPL, above the MAGI Medicaid limit." -us,scenario_093,dependent1_medicare_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model converted the bare disability flag into Medicare eligibility and assumed the SSDI waiting period was met. The prompt explicitly says unlisted facts are false, so SSDI entitlement and the 24-month Medicare disability pathway were not available for this 27-year-old dependent." -us,scenario_093,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model correctly named the SSDI-after-24-months pathway but then treated `is disabled` as equivalent to SSDI receipt and completed waiting-period entitlement. Because the facts only list disability and do not list SSDI, ESRD, ALS, or age 65, the Medicare eligibility test returns no for dependent1." -us,scenario_093,dependent1_medicare_eligible,glm-5.2,llm_error,health_coverage,False,The model misstated the PolicyEngine rule as age 65 or disability status alone. PolicyEngine does not make a 27-year-old Medicare eligible from disability status by itself; the required Medicare-qualifying medical or entitlement pathway is absent from the listed facts. -us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted $1,000 of personal auto-loan interest, reducing taxable income from $80,208.25 to $79,208.25. It then abandoned its own $8,509 calculation for an unsupported $8,067 adjustment instead of applying the traced 2026 brackets and $500 dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model improperly included the adult dependent's $45,000 wages in the married couple's joint return. It also used a $29,200 standard deduction and subtracted an invented $4,246 refundable-credit reconciliation from an output defined before refundable credits." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model rejected the disabled adult as a dependent by inferring that their wages established self-support, despite the supplied tax-unit relationship and the qualifying-child disability pathway. It therefore omitted the $500 nonrefundable dependent credit and also used an understated $31,500 standard deduction." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model applied the qualifying-relative gross-income test while overlooking the qualifying-child pathway for a permanently disabled person of any age. That mistake removed the $500 dependent credit, while its estimated $31,500 deduction and bracket thresholds further overstated tax." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's submitted $10,371 contradicts its own derivation of approximately $8,371 after the dependent credit. It also deducted the full $3,000 of student-loan interest instead of the applicable capped amount embedded in the $2,689.62 combined adjustments." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a projected $30,000 joint standard deduction instead of $32,200 and estimated bracket thresholds rather than the applicable 2026 values. It also denied the disabled adult's qualifying-child pathway and omitted the $500 nonrefundable credit." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly reached AGI near $112,408 but used an estimated $30,300 standard deduction and then submitted $10,800 despite calculating tax of $9,376. The unexplained upward adjustment has no tax computation behind it and omits the $500 dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,The model added the adult dependent's separately calculated single-return tax to the requested joint tax-unit output. It also deducted ESI premiums as pretax amounts even though the trace's employment income adjustment consists of the traditional 401(k) contributions. -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly treated 2026 as a post-TCJA reversion year with personal exemptions and smaller standard deductions. It also added a separate tax liability for the adult dependent and deducted listed ESI premiums from wages, neither of which belongs in this output." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model's $163,156 gross-income base includes the dependent's $45,000 wages in the married couple's joint return. The correct joint-return gross-income computation begins with the head and spouse's wages and interest, producing $115,097.88 after traditional 401(k) contributions." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a pre-TCJA-style 2026 regime with personal exemptions and combined a separate dependent return with the joint return. The applicable computation instead uses the $32,200 joint standard deduction and subtracts the $500 dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model summed tax for the married couple and a separate single return for the adult dependent, while the requested value is the traced tax-unit liability. Its $95,630 joint AGI also reflects improper deductions, rather than the $112,408.25 AGI used in the joint calculation." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model included a separately computed tax liability for the adult dependent in the requested amount. The output is derived from the married joint return alone, with the dependent affecting that return through the $500 nonrefundable credit." -us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model correctly recognized the $500 dependent credit but then added $3,353.74 from the dependent's separate return. It also used an estimated $30,840 standard deduction instead of $32,200, overstating the joint liability before that improper addition." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated $112,408.25 of AGI as fully sheltered by deductions and credits. After the $32,200 standard deduction, $80,208.25 remains taxable and produces $9,128.99 before the $500 nonrefundable credit." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The answer implies excessive deductions or credits reduced the traced $9,128.99 pre-credit liability by $2,781.99. The listed medical expenses do not generate an itemized deduction here, and the applicable nonrefundable dependent credit is $500, yielding $8,628.99." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model included the dependent's wages in the married filing jointly income base and then deducted all three people's ESI premiums as pretax wage exclusions. The traced joint return excludes the dependent's wages and reduces employment income only by $4,058.12 of traditional 401(k) contributions." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"Its $124,708 taxable-income figure necessarily includes income outside the married couple's joint return, chiefly the adult dependent's wages. The correct taxable income is $80,208.25, and the model also omitted the $500 nonrefundable dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The submitted liability is consistent with taxing the adult dependent's wages as part of the joint income base. The joint return instead has $80,208.25 of taxable income; after its $9,128.99 tentative tax, the stated $500 dependent credit yields $8,628.99." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model overstated taxable income by exactly $5,000, using $85,208 instead of $80,208.25. It therefore applied the right general credit structure to the wrong deduction base, failing to use the full $32,200 standard deduction and traced adjustments." -us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The amount implies that household wages were conflated with the married couple's joint-return wages or that substantial allowed deductions were omitted. The correct joint computation excludes the dependent's $45,000 wages, reaches $80,208.25 of taxable income, and subtracts the $500 dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model incorrectly imposed a post-TCJA-sunset regime with personal exemptions and added the adult dependent's separate single-return tax. It also omitted the $500 nonrefundable dependent credit from the married couple's liability. -us,scenario_093,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used post-TCJA rate assumptions that produced $11,001 of tax on $81,658, far above the applicable 2026 bracket calculation. It then added the dependent's separate tax and omitted the $500 credit allowed on the joint return." +us,scenario_093,dependent1_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated the disability flag as sufficient for Missouri's SSI-related or MO HealthNet for the Disabled pathway and explicitly disregarded the pathway's additional eligibility requirements. Dependent 1 receives no SSI and qualifies for no Medicaid category, while their 2.82-times-FPL MAGI also disqualifies them from the MAGI pathways." +us,scenario_093,dependent1_medicare_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model treated the listed disability flag as automatic Medicare eligibility and explicitly assumed completion of an SSDI waiting period despite the instruction that unlisted statuses are false. No SSDI receipt, 24-month entitlement period, ESRD, or ALS was listed, so the dependent does not meet an under-65 Medicare pathway." +us,scenario_093,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model correctly identified the SSDI-based 24-month rule but then replaced its required SSDI entitlement history with the generic disability flag. PolicyEngine does not equate disability status with Medicare eligibility, and the unlisted SSDI, ESRD, and ALS criteria are false." +us,scenario_093,dependent1_medicare_eligible,glm-5.2,llm_error,health_coverage,False,"The model misstated PolicyEngine's rule as age 65 or disability status. For someone age 27, the disability flag alone is insufficient; none of the required under-65 Medicare entitlement criteria was supplied." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $1,000 of auto-loan interest from taxable income even though the trace’s $32,200 standard deduction already determines taxable income of $80,208.25. It then abandoned its own $8,509 calculation and submitted $8,067 without a valid computation." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model included the adult dependent’s $45,000 wages in the married couple’s joint income, inflating AGI and taxable income. It also used a $29,200 standard deduction instead of $32,200 and subtracted an invented $4,246 reconciliation amount rather than the actual $500 nonrefundable dependent credit." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model rejected the disabled adult as a dependent based on an assumed self-support failure and therefore omitted the $500 nonrefundable dependent credit. It also used an understated $31,500 standard deduction instead of $32,200." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated the disabled adult’s $45,000 wages as disqualifying dependency and omitted the $500 nonrefundable credit. It also used an approximate $31,500 standard deduction instead of the applicable $32,200." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model’s stated inputs produce tax near $8,371 after its claimed credit, but it arbitrarily added $2,000 for the disabled adult’s separate tax and submitted $10,371. This output is only the joint tax unit’s liability and does not add a separate return for the dependent." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used projected 2026 parameters—a $30,000 standard deduction and approximate bracket thresholds—instead of the applicable $32,200 deduction and exact rates. It also denied the $500 nonrefundable dependent credit, leaving both taxable income and tax overstated." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly derived $112,408 of AGI but used an understated estimated standard deduction and then discarded its own $9,376 tax calculation to submit $10,800 without a computation. It also omitted the $500 nonrefundable dependent credit." +us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model summed a joint return and a separate single return for the adult dependent. The requested value is the joint tax unit’s liability alone, with the dependent generating a $500 nonrefundable credit rather than having separate tax added to this output." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model incorrectly assumed the TCJA rules expired for 2026 and used personal exemptions plus reduced standard deductions. It also summed a separate return for the adult dependent instead of computing only the joint tax unit’s liability. +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model treated all $163,000 of household wages as income on the married joint return, improperly including the adult dependent’s $45,000 wages. The joint return instead starts from the head’s and spouse’s wages and applies the $500 dependent credit." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a pre-TCJA 2026 regime with personal exemptions and combined the couple’s liability with a separate return for the dependent. The applicable computation uses the $32,200 joint standard deduction and reports only the joint tax unit’s post-nonrefundable-credit liability." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model summed the married couple’s return with a separate single return for the adult dependent. It also reduced wages by employer-sponsored insurance premiums and applied personal exemptions, neither of which appears in the traced derivation." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model added tax from a separate return for the adult dependent to the requested joint tax-unit output. The correct joint computation has $9,128.99 of pre-credit liability and subtracts the $500 dependent credit, with no separate-return tax included." +us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added $3,353.74 from a separate return for the adult dependent even though this output covers the joint tax unit. It also used an estimated $30,840 joint standard deduction instead of $32,200." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model incorrectly concluded that the standard and retirement deductions eliminate taxable income. They leave $80,208.25 of taxable income, producing $9,128.99 before credits and $8,628.99 after the $500 nonrefundable credit." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model’s $6,347 estimate reflects excessive deductions or an understated tax base. The listed adjustments and $32,200 standard deduction leave $80,208.25 taxable, and the model also incorrectly stated that no dependent credit applies." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model included part of the adult dependent’s wages in the joint return and separately subtracted employer-sponsored insurance premiums from the listed wages. The traced joint wage base excludes the dependent’s wages and is reduced only by $4,058.12 of pre-tax 401(k) contributions." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model’s $124,708 taxable income incorporates the adult dependent’s wages into the married joint return. It also omitted the $500 nonrefundable dependent credit; the correct joint taxable income is $80,208.25." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The submitted liability is consistent with taxing household-wide wages rather than only the head’s and spouse’s income on their joint return. Although the model recognized the $500 credit, it applied it to an inflated tax base instead of $80,208.25 of taxable income." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model overstated taxable income by $5,000, using $85,208 instead of $80,208.25. The correct $112,408.25 AGI less the $32,200 standard deduction produces $9,128.99 before the correctly recognized $500 credit." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The $13,980 estimate reflects an inflated tax base consistent with including the adult dependent’s wages in household taxable income. The requested joint tax unit excludes those wages and applies a $500 nonrefundable dependent credit." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied post-sunset rules with personal exemptions and then added a separate single return for the adult dependent. The applicable 2026 computation uses the $32,200 joint standard deduction, reports only the joint tax unit, and subtracts the $500 dependent credit." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model added $3,820 of tax from a separate return for the adult dependent to the joint liability. It also used an understated $30,750 standard deduction, incorrect tax brackets, and omitted the $500 nonrefundable dependent credit." +us,scenario_093,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model correctly approximated the joint taxable income and its $9,129 pre-credit tax, but then added roughly $3,220 from the dependent’s separate return. It also failed to subtract the $500 nonrefundable dependent credit from the joint liability." us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model placed the adult dependent's $45,000 wages inside the parents' married filing jointly return. It also omitted the $4,058.12 traditional 401(k) wage exclusion and incorrectly declared the small traditional IRA contributions fully phased out." -us,scenario_093,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The zero answer ignores the $80,208.25 of taxable income remaining after the traced adjustments and $32,200 standard deduction. The 2026 brackets produce $9,128.99 before the $500 nonrefundable dependent credit, not zero." -us,scenario_093,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $16,778 of employer insurance premiums from the couple's wages, allowed $3,000 rather than the traced student-loan-interest amount, and used estimated deductions and brackets. It also vacillated between adding the dependent's separate tax and reporting only the joint tax, finally submitting a number supported by neither calculation." +us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model included the adult dependent’s $45,000 wages in the married joint return, producing $129,456 of taxable income instead of $80,208.25. It also failed to reduce wages by the head’s and spouse’s $4,058.12 of pre-tax 401(k) contributions and wrongly denied their IRA deductions." +us,scenario_093,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The zero answer ignores the $80,208.25 of taxable income remaining after the applicable adjustments and standard deduction. That income generates $9,128.99 of tax before the $500 nonrefundable credit." +us,scenario_093,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-deducted employer-sponsored insurance premiums from wages, allowed $3,000 rather than $2,499.62 of student-loan-interest deduction, and omitted the $190 IRA deduction. It then submitted $7,625.75 despite its own competing calculations of $7,248, $9,357, and $9,345, and it failed to apply the $500 dependent credit." +us,scenario_093,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model’s stated regular liability of about $13,491 does not reconcile to its submitted $3,778, so the final value is an unsupported arithmetic departure from its own reasoning. It also wrongly included the adult dependent’s wages in the joint return and explicitly refused to subtract the $500 nonrefundable dependent credit even though the requested metric requires it." us,scenario_093,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_093,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly computed all six individual FICA components, which sum to $12,469.50, but then misstated their sum as $11,469.50 and submitted an unrelated $12,348.76. This is a final arithmetic and answer-transcription error." -us,scenario_093,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model's stated Social Security and Medicare totals correctly add to $12,469.50, but it submitted $12,487.50. It transposed digits when transferring the correctly derived result into the output." -us,scenario_093,payroll_tax,claude-sonnet-5,llm_error,other,False,"The six payroll-tax components listed in the explanation sum to $12,469.50, not $12,622.90. The model made a final addition error despite computing every individual Social Security and Medicare amount correctly." -us,scenario_093,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"Applying the model's stated 7.65% combined FICA rate to $163,000 yields $12,469.50, not $12,481. The model performed the multiplication incorrectly." -us,scenario_093,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model deducted all three $8,389 employer-sponsored insurance premium amounts from FICA wages without an input establishing employee pre-tax Section 125 salary reductions. Payroll tax therefore applies to the full $163,000 of listed gross wages, yielding $12,469.50." -us,scenario_093,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model treated each $8,389 employer-sponsored insurance premium as a pre-tax employee payroll deduction and reduced FICA wages to $137,833. Those premiums do not reduce the listed gross-wage payroll-tax base, so FICA applies to the full $163,000." -us,scenario_093,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly subtracted $8,389 of employer-sponsored insurance premiums from each worker's FICA wages. With no employee pre-tax premium deduction specified, Social Security and Medicare taxes apply to all $163,000 of gross wages." -us,scenario_093,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model asserted that it applied Social Security and Medicare taxes to all three wage amounts but miscomputed the total. The stated wage base of $163,000 at 7.65% produces $12,469.50, not $12,300.30." -us,scenario_093,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's displayed formula equals 7.65% of $163,000, which is $12,469.50. Its $9,024 output is an arithmetic failure inconsistent with its own formula and wage inputs." -us,scenario_093,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model incorrectly classified the three employer-sponsored insurance premium amounts as employee pre-tax deductions from FICA wages. The specified premiums do not reduce gross wages for this calculation, leaving a $163,000 payroll-tax base and $12,469.50 of employee FICA." -us,scenario_093,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so the required output was missing." -us,scenario_093,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The $0 answer omits employee Social Security and Medicare taxes on all three workers' wages. The $163,000 wage base generates $10,106 of Social Security tax and $2,363.50 of Medicare tax." -us,scenario_093,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly calculated the three worker amounts as $4,054.50, $4,972.50, and $3,442.50, which total $12,469.50, but submitted $12,553.50. This is a final answer-transcription error after a correct derivation." -us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model reached approximately the correct $79,800 taxable-income base but applied an inaccurate shortcut for Missouri's graduated brackets. Applying the 2026 schedule to $79,776.80 yields $3,389.65, not $3,506." -us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used $119,156 as AGI, omitted the traced above-the-line and Missouri deductions, and then substituted an effective-rate estimate for the statutory schedule. Missouri taxable income is $79,776.80, producing $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model overstated Missouri AGI at $116,466 by mishandling the wage and above-the-line adjustments, then omitted the full traced Missouri deduction calculation. The correct Missouri AGI is $112,408.25 and taxable income is $79,776.80." -us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model's own taxable-income estimate was close to the traced $79,776.80, but its bracket arithmetic is internally wrong: its stated components total about $3,640, not $4,126. The actual 2026 Missouri schedule yields $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly added a separate Missouri tax for the disabled 27-year-old household member. The trace exempts that member from Missouri income tax, so only the two primary earners' allocated liability of $1,453.04 and $1,936.62 is included." -us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated all listed employer-sponsored insurance premiums as Section 125 exclusions despite no such pre-tax treatment being specified, reducing AGI by $16,778. It also invented a $4,400 Missouri personal exemption and deducted the full estimated federal tax instead of the traced $431.45 federal-income-tax deduction." -us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model asserted a Missouri taxable-income base near $102,000 without deriving it from the household inputs. The traced deductions reduce Missouri AGI of $112,408.25 to taxable income of $79,776.80." -us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model calculated a second, single-filer Missouri liability of $1,056.06 for the disabled third member. That member is exempt from Missouri income tax, and the applicable calculation uses the 2026 rules rather than 2024 brackets." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an estimated 4.8% top rate instead of the applicable 2026 Missouri schedule and did not establish the traced $79,776.80 taxable-income base. Those rules yield $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies Missouri rates were applied to a substantially overstated base rather than the traced taxable income of $79,776.80. Applying the 2026 brackets to that base yields $3,389.65, not $6,055." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an unspecified combined household-income base and a rough 4.5% effective rate. It failed to derive Missouri taxable income of $79,776.80 after the applicable adjustments and deductions." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Although the model named the relevant deduction categories, its $4,166 result implies that it did not apply their traced amounts. Missouri AGI of $112,408.25 is reduced to $79,776.80 before the rate schedule is applied." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's near-reference result still reflects inaccurate deduction or bracket arithmetic. The exact traced base is $79,776.80 and the 2026 Missouri schedule produces $3,389.65, not $3,421." -us,scenario_093,state_income_tax_before_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model added $30,400 of taxable income for a separate single-filer return by the disabled third household member, who is exempt from Missouri income tax in the trace. It also treated Missouri's schedule as a flat 4.7% tax instead of applying the graduated brackets." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly claimed the deductions and credits eliminate the entire Missouri tax base. The deductions leave $79,776.80 of Missouri taxable income, which generates $3,389.65 before refundable credits." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model supplied only a rough estimate and did not apply the exact Missouri deduction and bracket calculation. The statutory schedule on traced taxable income of $79,776.80 yields $3,389.65 rather than $3,427." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model improperly included the disabled third member's $45,000 wages in the couple's federal AGI, producing $157,598 instead of Missouri AGI of $112,408.25. It then invented personal/dependent exemptions and a health-insurance subtraction that are not part of the traced computation." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model's $124,708 taxable-income estimate incorporates income outside the liable joint return and omits required Missouri deductions. The liable base is $79,776.80, while the disabled third member has no separate Missouri tax." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model overstated the tax base by aggregating income beyond the liable joint return and incorrectly declared the federal-income-tax deduction unavailable. The trace allows a $431.45 federal-tax deduction and leaves $79,776.80 taxable." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model introduced unspecified exemptions and used the wrong amount for the federal-income-tax deduction, understating the taxable base or tax. The traced deductions leave $79,776.80 and the schedule yields $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model stopped at a rough estimate after the standard deduction and did not compute the complete Missouri deduction stack or statutory brackets. The exact taxable base of $79,776.80 yields $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model imposed approximately $1,547 of separate Missouri tax on the disabled third household member. That member is exempt, and the model also overstated the couple's taxable income relative to the traced $79,776.80." -us,scenario_093,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model incorrectly added $1,354 of Missouri tax for the disabled third member and used a 4.95% top-rate schedule. The trace assigns no Missouri liability to that member and applies the 2026 schedule to $79,776.80." -us,scenario_093,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model invented a $13,000 Missouri standard deduction, $6,000 of personal exemptions, and a percentage-based federal-tax deduction while also using obsolete estimated brackets. The trace instead uses the $32,200 standard deduction, $431.45 federal-income-tax deduction, and other allowable deductions to reach $79,776.80." -us,scenario_093,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model incorrectly treated all $161,656 of wages and interest, including the disabled third member's wages, as joint-return federal AGI. Missouri AGI for the liable return is $112,408.25, and the third member is exempt from Missouri tax." -us,scenario_093,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer implies that the model treated deductions or exemptions as eliminating Missouri taxable income. The traced computation leaves $79,776.80 taxable and produces $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model improperly included the disabled third member's $45,000 wages in the married joint return and used obsolete deduction and rate figures. Its final $11,145.20 also contradicts its own stated approximate tax of $6,809; the liable taxable base is $79,776.80 and yields $3,389.65." -us,scenario_095,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the full $14,022 of Social Security retirement benefits in AGI, raising AGI from $15,286.89 to about $29,605. It then taxed income that PolicyEngine excludes before the standard deduction because the household's provisional income is below the Social Security taxation threshold." -us,scenario_095,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model got the AGI concept mostly right but used a post-TCJA-sunset standard deduction of $9,800 instead of the applicable 2026 standard deduction for this elderly taxpayer. That understated deduction created $5,486 of taxable income, while the correct deduction wipes out the $15,286.89 AGI." -us,scenario_095,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model correctly recognized that Social Security is not taxable, but then failed to remove the nontaxable Social Security amount from AGI before applying the standard deduction. Its taxable-income figure is consistent with subtracting the standard deduction from an income total that still includes the full Social Security benefit, rather than from the $15,286.89 AGI." +us,scenario_093,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly listed all six employee FICA components, which sum to $12,469.50, but then stated an intermediate total of $11,469.50 and submitted the unrelated value $12,348.76. Its submitted output does not follow its own component calculation." +us,scenario_093,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model explicitly derived $10,106 of Social Security tax plus $2,363.50 of Medicare tax, totaling $12,469.50, but submitted $12,487.50. It introduced an unexplained $18 transposition error after completing the calculation correctly." +us,scenario_093,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The six amounts the model listed sum to $12,469.50, not $12,622.90. It made a $153.40 addition error while aggregating otherwise correct person-level Social Security and Medicare taxes." +us,scenario_093,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"Applying the model's stated 7.65% combined FICA rate to $163,000 yields $12,469.50, not $12,481.00. The submitted result contains an $11.50 arithmetic error." +us,scenario_093,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly subtracted all three listed employer-sponsored insurance premium amounts from FICA wages. Those inputs do not establish employee pre-tax Section 125 salary reductions, so payroll tax applies to the full $163,000 of wages and totals $12,469.50." +us,scenario_093,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model treated each $8,389 employer-sponsored insurance premium as a pre-tax employee wage deduction, reducing FICA wages from $163,000 to $137,833. The listed premiums do not reduce the employees' payroll-tax base, so FICA is calculated on the full gross wages." +us,scenario_093,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model incorrectly deducted $8,389 of employer-sponsored insurance premiums from each worker's FICA wages. These inputs do not represent employee Section 125 contributions, leaving the full $163,000 subject to employee Social Security and Medicare taxes." +us,scenario_093,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model applied the employee payroll-tax rates incorrectly: 7.65% of the stated $163,000 of wages is $12,469.50, not $12,300.30. Its answer understates FICA by $169.20 despite correctly identifying the applicable components." +us,scenario_093,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's own formula, 6.2% plus 1.45% on $163,000, evaluates to $12,469.50 rather than $9,024. It made a major arithmetic error when evaluating the stated FICA calculation." +us,scenario_093,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model incorrectly treated the listed employer-sponsored insurance premiums as employee pre-tax salary reductions and removed $25,167 from FICA wages. Payroll tax instead applies to all $163,000 of gross wages because no such employee wage deductions were supplied." +us,scenario_093,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required output contract." +us,scenario_093,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer omits employee Social Security and Medicare taxes on all three workers' $163,000 of wages. Those wages generate $10,106 of Social Security tax and $2,363.50 of Medicare tax." +us,scenario_093,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model correctly calculated person-level FICA amounts of $4,054.50, $4,972.50, and $3,442.50, which sum to $12,469.50, but submitted $12,553.50. It added an unexplained $84 after deriving the correct total." +us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model reached taxable income close to $79,776.80 but converted it to tax using an approximate 4.7% calculation and an invented graduated-bracket adjustment. Applying Missouri's exact schedule to that base yields $3,389.65, not $3,506." +us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used $119,156 as AGI, omitted the $2,689.62 of above-the-line deductions and Missouri's full deduction calculation, and relied on an approximate effective rate. The correct taxable-income base is $79,776.80." +us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model overstated Missouri AGI at $116,466 by using an incorrect federal adjustment calculation, then substituted an estimated $30,000 deduction. The trace uses Missouri AGI of $112,408.25 and the applicable Missouri deductions to produce $79,776.80 of taxable income." +us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model's approximate taxable income was close to the correct $79,776.80, but its bracket arithmetic is internally wrong: its stated base-plus-top-bracket computation does not support $4,126. Missouri's exact graduated schedule yields $3,389.65." +us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly added about $1,400 of separate Missouri tax for the 27-year-old household member. This output assigns the liability across the two primary earners and exempts the third member, so only the $3,389.65 joint-unit result belongs in the household total." +us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly treated all listed employer-sponsored insurance premiums as pre-tax wage reductions, invented a $4,400 Missouri personal exemption, and deducted the entire estimated federal tax liability. The trace instead gives Missouri AGI of $112,408.25 and taxable income of $79,776.80, including only $431.45 for the Missouri federal-income-tax deduction." +us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model asserted approximately $102,000 of Missouri taxable income without reconciling that figure to its stated deductions. The correct Missouri taxable income is $79,776.80 after the traced above-the-line and Missouri deductions." +us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model used obsolete 2024 parameters and added $1,056.06 of tax for a separate single-filer unit. The third household member is exempt in this output, and the applicable 2026 calculation produces $3,389.65 across the two primary earners." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an estimated 4.8% top rate instead of the applicable Missouri schedule and did not derive the traced $79,776.80 taxable-income base. Those exact brackets applied to that base yield $3,389.65." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $6,055 does not reflect the traced Missouri taxable income of $79,776.80. It is consistent with applying Missouri rates to a substantially overstated base without the required above-the-line and Missouri deductions." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a rough 4.5% household estimate rather than calculating the exact graduated tax on $79,776.80. The applicable schedule produces $3,389.65." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model named the relevant deduction categories but did not apply the traced amounts, producing a liability consistent with an overstated taxable-income base. Missouri taxable income is $79,776.80 after the exact deductions, and its tax is $3,389.65." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The answer is $31.35 above the exact result because the model used an approximate deduction or bracket calculation rather than the traced amounts. Applying the exact Missouri schedule to $79,776.80 yields $3,389.65." +us,scenario_093,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added the 27-year-old's wages as a separately taxed single-filer unit and applied a flat 4.7% rate to combined taxable income. This output exempts that third member, while Missouri's graduated schedule applies to $79,776.80 for the joint unit." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model falsely concluded that deductions and allowances eliminated all taxable income. The traced deductions leave $79,776.80 of Missouri taxable income, producing $3,389.65 before refundable credits." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model used a generic approximation instead of the exact Missouri bracket computation. Taxing the traced $79,776.80 base under the applicable schedule produces $3,389.65, not $3,427." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model incorrectly included the third household member's $45,000 of wages in a purported joint federal AGI of $157,598. The relevant joint unit has IRS gross income of $115,097.88, and the third member is exempt from Missouri tax in this output." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model overstated Missouri taxable income at $124,708 by failing to reproduce the traced income unit and deductions. The correct taxable-income base is $79,776.80." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model treated the federal-income-tax deduction as unavailable and used a substantially overstated income base. The Missouri calculation includes a $431.45 federal-income-tax deduction and reaches $79,776.80 of taxable income." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model understated the result by applying unspecified exemptions and an excessive federal-income-tax deduction. The applicable federal-tax deduction is $431.45, and the full traced calculation leaves $79,776.80 taxable." +us,scenario_093,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model reduced the calculation to an approximate tax after a standard deduction and omitted the exact graduated-schedule computation. The traced $79,776.80 taxable base produces $3,389.65." +us,scenario_093,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model created separate couple and dependent tax liabilities and added $1,547 for the third household member. That member is exempt in this output, so the household result consists only of the $3,389.65 allocated to the two primary earners." +us,scenario_093,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model incorrectly calculated a separate $1,354 Missouri liability for the third household member and used a 4.95% schedule. The third member is exempt here, and the applicable schedule on the joint unit's $79,776.80 taxable income yields $3,389.65." +us,scenario_093,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model added approximately $1,047 of separate single-filer tax for the third household member. This output exempts that member and allocates the entire $3,389.65 liability across the two primary earners." +us,scenario_093,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model invented a $13,000 Missouri standard deduction, $6,000 of personal exemptions, and a percentage-based federal-tax deduction exceeding the traced $431.45 amount. Missouri uses the traced deductions to reach $79,776.80 of taxable income, then applies the applicable graduated schedule." +us,scenario_093,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model combined all three adults' income into a $161,656 joint AGI even though the third adult's wages are outside the joint tax unit. The relevant IRS gross income is $115,097.88, and the third member is exempt from Missouri income tax in this output." +us,scenario_093,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer implies that the model treated the household's income as fully sheltered or never performed the Missouri calculation. The traced deductions leave $79,776.80 taxable and therefore $3,389.65 of tax." +us,scenario_093,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly included the dependent adult's $45,000 of wages in the married joint return, then contradicted its own $6,809 intermediate estimate by submitting $11,145.20. The relevant joint unit starts from $115,097.88 of IRS gross income, and the third member is exempt in this output." +us,scenario_093,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model started from an unsupported $124,942 taxable-income estimate and applied an approximate 4.5% rate. The traced deductions instead produce $79,776.80 of Missouri taxable income, on which the exact tax is $3,389.65." +us,scenario_095,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the full $14,022 of Social Security in AGI and then applied an age-65 single-filer standard deduction. Social Security is not taxable at this provisional-income level, and the surviving-spouse filing-status deduction eliminates the resulting $15,286.89 AGI." +us,scenario_095,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model correctly reached approximately $15,286 of AGI but treated the head as a single filer and applied a $9,800 deduction. The surviving-spouse filing status provides the larger applicable standard deduction, reducing taxable income to zero." +us,scenario_095,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model correctly concluded that none of the Social Security is taxable, but then retained it in its $29,308 AGI and computed taxable income from that inflated figure. Excluding the nontaxable Social Security produces $15,286.89 of AGI, which the surviving-spouse standard deduction fully offsets." +us,scenario_095,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a $1,200 refundable senior credit based solely on the head being age 73. No such refundable federal credit applies here; with no qualifying children or education facts and income above the childless EITC range, every refundable-credit component equals zero." us,scenario_095,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_095,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_095,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated being age 73 as automatically qualifying the head for New Jersey's aged Medicaid category and then assumed a high enough aged income standard. PolicyEngine's category computation assigns `medicaid_category = NONE`, so the head does not qualify through the aged pathway, and the gross income of about $29,605 at 1.84 FPL is not rescued by any standard Medicaid category." -us,scenario_095,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented an ABD or medically needy eligibility route by excluding most Social Security, deducting self-employment tax and medical expenses, and applying an asset screen. PolicyEngine does not place this person in any aged, blind, disabled, medically needy, or MAGI category; with `medicaid_category = NONE`, the low $500 asset amount and small medical expenses do not create eligibility." -us,scenario_095,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used the shortcut that an elderly New Jersey resident with low assets and modest income is Medicaid eligible. The correct computation first requires a qualifying Medicaid category, and PolicyEngine assigns none; the head's 1.84 FPL MAGI income and age do not satisfy any applicable pathway." -us,scenario_095,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable `head_medicaid_eligible` output or supporting explanation. This is a contract failure rather than a substantive Medicaid-rule calculation. +us,scenario_095,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age 73 as establishing an aged Medicaid pathway and then asserted a higher New Jersey income standard without identifying or applying a qualifying category. The person qualifies through no Medicaid category, so the $500 asset balance cannot produce eligibility." +us,scenario_095,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented eligibility through New Jersey's ABD or medically needy pathways by excluding most Social Security income and deducting medical expenses, even though the person qualifies through neither pathway. Its asset-limit analysis is immaterial because the engine assigns Medicaid category NONE." +us,scenario_095,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used elderly status, low assets, and a generic characterization of income as a substitute for testing a specific Medicaid pathway. Age 73 alone does not confer Medicaid eligibility, and this person qualifies through no Medicaid category." +us,scenario_095,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric value or explanation for head_medicaid_eligible, violating the required output contract." +us,scenario_095,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The unexplained affirmative answer treats the household facts as sufficient without applying categorical eligibility. The correct derivation assigns Medicaid category NONE and therefore yields 0. us,scenario_095,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_095,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_095,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model put self-employment income into the employee-side payroll_tax base and computed a FICA-like amount on it. Self-employment income belongs in the separate self_employment_tax output, while payroll_tax applies to wages and mandatory employee state payroll taxes; with no wages listed, payroll_tax is $0." -us,scenario_095,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model treated self-employment earnings as if they were employee wages subject to employee Social Security and Medicare taxes in payroll_tax. Its submitted $1,167.84 also contradicts its own stated $321.15 calculation, but the substantive rule error is that self-employment tax is excluded from payroll_tax and reported separately." +us,scenario_095,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model treated self-employment income as subject to employee-side payroll tax and placed an estimated SECA amount in payroll_tax. Because no wages are listed, the employee Social Security and Medicare tax base is zero; any tax on the $4,198 belongs exclusively in self_employment_tax." +us,scenario_095,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model incorrectly applied the employee Social Security and Medicare rates to self-employment income, which belongs in the separate self_employment_tax calculation. It also submitted $1,167.84 despite deriving $321.15 in its explanation, but both figures wrongly assign self-employment tax components to payroll_tax instead of using the zero wage base." us,scenario_095,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_095,self_employment_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model identified the correct 92.35% net-earnings factor and 15.3% combined rate, but its submitted value contradicts its own arithmetic. It rounded or transcribed the product incorrectly, submitting $594.29 instead of carrying the net-earnings calculation to the engine-rounded $593.23." -us,scenario_095,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model used the right general rule but replaced the actual net-earnings calculation with a rough estimate. It did not carry $4,198 through the 92.35% self-employment net earnings factor and 15.3% tax rate to the precise liability of $593.23." -us,scenario_095,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model applied the 15.3% rate to the full $4,198 self-employment income instead of first multiplying by the 92.35% net-earnings factor. It also treated the half self-employment tax deduction as affecting the self-employment tax liability itself, when that deduction affects income tax and not the SE tax computation." -us,scenario_095,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable self_employment_tax output or explanation. This is a missing-output failure rather than a substantive tax-rule calculation. -us,scenario_095,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated age 73 as bypassing the controlling income eligibility limit. Its own allotment calculation produced a 30% contribution exceeding the one-person maximum allotment, which yields $0 rather than a “modest benefit” of $2,363." -us,scenario_095,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly placed the household on an elderly net-income eligibility pathway despite $29,605 of annual income exceeding the applicable SNAP limit. It then awarded a minimum allotment even though minimum benefits do not convert an income-ineligible household into an eligible one." -us,scenario_095,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly asserted that $29,605 falls below New Jersey’s applicable 2026 SNAP income threshold and consequently invoked expanded categorical eligibility. It also substituted a purported $95 monthly state minimum for the allotment calculation, producing $1,140 instead of terminating eligibility at $0." -us,scenario_095,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model labeled the household’s countable income “very low” and inferred eligibility primarily from the elderly status and $500 of assets, omitting the controlling income-limit test. Applying that test to $29,605 yields no SNAP eligibility and therefore no annual benefit." -us,scenario_095,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model incorrectly treated New Jersey broad-based categorical eligibility and a 200% FPL comparison as sufficient to qualify this household. Because the household fails the applicable income eligibility test, the $24 minimum allotment cannot be applied and annual SNAP is $0." -us,scenario_095,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model treated the surviving spouse as a single filer and proceeded to exemptions and the 1.4% bracket. A qualifying surviving spouse has a $20,000 New Jersey gross-income filing threshold, and the household's $15,583 of New Jersey gross income is below it, producing zero tax." -us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model constructed taxable income from a federal-style AGI and deductions without first applying New Jersey's filing threshold for a qualifying surviving spouse. New Jersey gross income of $15,583 is below that status's $20,000 threshold, so no bracket tax is computed." -us,scenario_095,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model correctly excluded Social Security and the retirement distribution but incorrectly taxed the remaining income after exemptions and medical deductions. It omitted the $20,000 New Jersey gross-income threshold for a qualifying surviving spouse, which eliminates liability because gross income is only $15,583." -us,scenario_095,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model applied exemptions and the 1.4% bracket as though the filer were subject to tax on any positive taxable income. The surviving-spouse filing status carries a $20,000 New Jersey gross-income threshold, and $15,583 falls below it." -us,scenario_095,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model ignored the surviving-spouse filing threshold and taxed $2,198 after the retirement exclusion and personal exemptions. Because New Jersey gross income is $15,583, below the $20,000 threshold for a qualifying surviving spouse, the tax is zero before those downstream calculations." -us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model calculated bracket tax on remaining self-employment income after exemptions and a medical deduction. It failed to apply the qualifying-surviving-spouse $20,000 New Jersey gross-income threshold, under which the household's $15,583 yields no liability." -us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model proceeded from the retirement exclusion to personal, age, and medical deductions and then applied the first tax bracket. It omitted the controlling $20,000 gross-income threshold for a qualifying surviving spouse; $15,583 is below that threshold." -us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model invented a residual liability from a low-income tax reduction instead of applying the surviving-spouse filing threshold. New Jersey gross income of $15,583 is below the applicable $20,000 threshold, so the liability is eliminated entirely." -us,scenario_095,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model taxed income remaining after the pension exclusion, exemptions, and medical deduction. It missed that a qualifying surviving spouse owes no New Jersey income tax when gross income is below $20,000, as this household's $15,583 is." -us,scenario_095,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model used federal AGI, only one exemption, and a 1.75% rate, but the decisive error occurred earlier: it never applied the qualifying-surviving-spouse gross-income threshold. The household's $15,583 of New Jersey gross income is below $20,000, so no state tax is calculated." -us,scenario_095,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, violating the required output contract." -us,scenario_095,state_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model treated the filer as single, introduced a federal half-self-employment-tax deduction and a supposed standard deduction, and then applied the 1.4% bracket. The qualifying-surviving-spouse $20,000 New Jersey gross-income threshold controls first, making tax zero on $15,583 of gross income." -us,scenario_095,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model counted New Jersey ANCHOR homeowner property relief as a refundable state individual income tax credit. PolicyEngine's state_refundable_credits output aggregates refundable state tax credits, and this New Jersey household has no qualifying refundable state income tax credit, so adding a $1,500 ANCHOR amount puts a property-benefit program into the wrong output." -us,scenario_095,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not submit a parseable value or explanation for state_refundable_credits. The required output was the numeric annual amount, and the correct derivation yields $0 because New Jersey refundable state credits evaluate to zero for this household." +us,scenario_095,self_employment_tax,claude-fable-5,llm_error,other,False,"The model’s own calculation produced approximately $593.16, but it submitted $594.29 without any arithmetic supporting that value. It failed to carry the self-employment-tax base, component calculations, and rounding consistently to the submitted result of $593.23." +us,scenario_095,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,The model replaced the statutory self-employment-tax computation with an unsupported approximation of “about $591.” Applying the self-employment-tax earnings adjustment and the Social Security and Medicare components with prescribed rounding yields $593.23. +us,scenario_095,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model treated the full $4,198 as the tax base and then incorrectly invoked the deduction for one-half of self-employment tax as an adjustment to the liability. That deduction affects income-tax AGI, while the self-employment-tax calculation uses the statutory adjusted earnings base and yields $593.23." +us,scenario_095,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output or explanation, so the required value was missing." +us,scenario_095,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model correctly recognized that $4,198 exceeds the $400 threshold but then wrongly treated the one-half self-employment-tax deduction and unspecified offsets as eliminating the tax. The deduction reduces adjusted gross income rather than self-employment tax, so the liability remains $593.23." +us,scenario_095,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly exempted this elderly household from the applicable gross-income eligibility screen and proceeded directly to net-income deductions. Its own estimated 30% contribution of about $660 per month exceeds the roughly $292 maximum allotment, which yields $0 rather than the asserted $2,363 annual benefit." +us,scenario_095,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model incorrectly routed the household through an elderly net-income-only pathway after counting $29,605 of annual gross income. That income fails the applicable gross-income eligibility limit, so neither medical and earned-income deductions nor a minimum allotment can create the asserted $300 benefit." +us,scenario_095,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied an incorrect 200%-of-FPL broad-based categorical eligibility threshold and then invented a $95 monthly New Jersey minimum SNAP benefit. The household’s $29,605 income exceeds the applicable gross-income limit, and the minimum allotment applies only to an otherwise eligible household." +us,scenario_095,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled the household’s countable income “very low” without reconciling that claim with the listed $29,605 annual income. That income exceeds the applicable one-person gross-income limit, so the $500 asset balance cannot establish eligibility and the correct allotment calculation yields $0." +us,scenario_095,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly treated $2,467 in monthly gross income as below New Jersey’s applicable broad-based categorical eligibility limit. Because the household fails the gross-income screen, it is ineligible and cannot receive the $24 monthly minimum allotment, which is reserved for otherwise eligible households." +us,scenario_095,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model treated the surviving spouse as a single filer and proceeded to tax $2,198 at 1.4%. A qualifying surviving spouse with $15,583 of New Jersey gross income is below New Jersey’s $20,000 filing threshold, so the liability is zero before exemptions or bracket calculations matter." +us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model constructed taxable income from federal AGI and deductions, then applied the 1.4% bracket without enforcing the qualifying-surviving-spouse filing threshold. New Jersey gross income of $15,583 is below the applicable $20,000 threshold, producing zero tax." +us,scenario_095,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model correctly excluded Social Security and the retirement distribution but incorrectly taxed the residual self-employment income. It omitted the $20,000 New Jersey gross-income threshold for a qualifying surviving spouse, which eliminates the liability at $15,583 of gross income." +us,scenario_095,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model applied exemptions and the 1.4% bracket as though this filer were required to calculate tax on residual income. The surviving-spouse filing status carries a $20,000 New Jersey gross-income threshold, and $15,583 falls below it." +us,scenario_095,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model stopped after deriving $2,198 of income following the retirement exclusion and personal exemptions and taxed it at 1.4%. It failed to apply the qualifying-surviving-spouse $20,000 gross-income threshold, under which the household owes no New Jersey income tax." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model calculated medical and personal deductions against residual self-employment income instead of first applying the filing-status threshold. As a qualifying surviving spouse with $15,583 of New Jersey gross income, the filer is below the $20,000 threshold and has zero liability." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model applied retirement, personal, age, and medical deductions and then taxed the remainder. It omitted the qualifying-surviving-spouse $20,000 New Jersey gross-income threshold, which makes the tax zero at $15,583." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model invoked a low-income tax reduction but still retained $18.46 of liability. The controlling rule is the $20,000 gross-income filing threshold for a qualifying surviving spouse, which reduces liability to zero because New Jersey gross income is $15,583." +us,scenario_095,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model taxed income remaining after the pension exclusion, medical deduction, and exemptions. It failed to apply the $20,000 New Jersey gross-income threshold for qualifying surviving spouses, under which $15,583 produces no tax." +us,scenario_095,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model used federal AGI, only one exemption, and an incorrect 1.75% rate, but the prior controlling error was omitting the surviving-spouse filing threshold. New Jersey gross income of $15,583 is below the qualifying-surviving-spouse threshold of $20,000, so no bracket rate applies." +us,scenario_095,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model correctly identified major New Jersey exclusions and deductions but then applied the 1.4% bracket to approximately $2,010. It overlooked the $20,000 gross-income threshold attached to qualifying-surviving-spouse status, which yields zero tax at $15,583." +us,scenario_095,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, violating the required submission contract." +us,scenario_095,state_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model deducted half of self-employment tax and a single $1,000 amount before applying the 1.4% bracket, bypassing the filing-status threshold. The filer is a qualifying surviving spouse whose $15,583 of New Jersey gross income is below the $20,000 threshold, so the liability is zero." +us,scenario_095,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model incorrectly classified a $1,500 NJ ANCHOR homeowner property-tax benefit as a refundable state individual income tax credit. ANCHOR is outside `nj_refundable_credits`, which yields $0 for this household." +us,scenario_095,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model omitted the required `state_refundable_credits` output entirely. The required computation evaluates `nj_refundable_credits` at $0 and therefore yields a $0 statewide refundable-credit aggregate. us,scenario_095,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_098,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly put the full $30,595 of Social Security retirement income into AGI, producing $35,035 instead of the $4,440 AGI from taxable pension income minus the farm rent loss. It also invoked personal exemptions, which do not apply for federal individual income tax in 2026, and therefore computed tax on income that the federal taxable-Social-Security and standard-deduction steps eliminate." +us,scenario_098,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated all $30,595 of Social Security as taxable when it set AGI to $35,035. After the farm rental loss offsets most of the pension, provisional income is below the Social Security taxation threshold, leaving $4,440 of AGI that the standard deduction fully eliminates." +us,scenario_098,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model counted all Social Security and the $1,100 energy subsidy as taxable income, omitted the $11,100 farm rental loss, and used an inapplicable $13,500 standard deduction. The farm loss leaves $4,440 of non-Social-Security income, Social Security remains nontaxable under the provisional-income test, and the standard deduction reduces taxable income to zero; its claimed $950 age credit does not repair those upstream errors." us,scenario_098,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_098,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_098,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age 80 as placing the head in an Illinois aged Medicaid pathway and then relied on negative farm rent income to pass an income test. PolicyEngine found no Medicaid category match at all, so the aged pathway never became available and the MAGI income level of 2.20 FPL could not override that categorical failure." -us,scenario_098,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly applied Illinois ABD and medically needy spend-down eligibility as available pathways for this head. PolicyEngine's trace assigns medicaid_category = NONE and il_hbi_eligible = False, so medical expenses and negative farm rent income never convert the household into a qualifying Medicaid category." -us,scenario_098,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model shortcut age 80 plus perceived low countable income and low assets into Medicaid eligibility. The correct derivation first checks whether any Medicaid category applies, and PolicyEngine found none; income and resources therefore do not establish eligibility." -us,scenario_098,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable head_medicaid_eligible value or supporting explanation. This is a missing-output failure rather than a substantive Medicaid rule calculation. +us,scenario_098,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age 80 and its own net-income calculation after the farm-rent loss as sufficient for an aged Medicaid pathway. PolicyEngine requires qualification through a specific category, and this person fails HBI eligibility overall and matches no Medicaid category despite passing HBI's age and resource tests." +us,scenario_098,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented eligibility through an Illinois medically needy spend-down by subtracting the listed medical costs and farm-rent loss, without establishing that PolicyEngine assigned that pathway or that the expenses satisfied its spend-down rules. The person instead fails HBI eligibility and matches no Medicaid category." +us,scenario_098,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model converted old age, asserted low countable income, and assets into automatic Medicaid eligibility without identifying a qualifying pathway. Passing age and resource tests is not enough: the person fails HBI eligibility overall, receives no SSI, and matches no Medicaid category." +us,scenario_098,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for head_medicaid_eligible, violating the required output contract." +us,scenario_098,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated a purportedly low countable MAGI as establishing aged, blind, or disabled Medicaid eligibility. MAGI at 2.20× FPL is relevant only after assignment to a MAGI category, while this person matches no category and fails HBI eligibility overall." us,scenario_098,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_098,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_098,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_098,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_098,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model asserted that an elderly household faces no SNAP gross-income test and proceeded directly to medical and uncapped shelter deductions. PolicyEngine applies the applicable gross-income limit here, and the household's approximately $35,035 annual income fails it, so the minimum allotment never applies." -us,scenario_098,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used deductions to estimate net income below the net-income limit but omitted the prior gross-income eligibility test. Because gross income exceeds the applicable one-person limit, the household is ineligible and cannot receive the $23 monthly minimum allotment." -us,scenario_098,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model characterized the household as very low income despite roughly $35,035 of annual income after the farm rental loss and did not test that amount against the SNAP gross-income limit. The household fails that threshold, making its estimated positive benefit invalid." -us,scenario_098,snap,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model allowed the large rent and shelter deductions to reduce SNAP net income to zero without first applying the gross-income eligibility screen. The household exceeds the applicable gross-income limit, so shelter deductions cannot establish eligibility or yield the maximum allotment." -us,scenario_098,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no SNAP value or explanation, violating the required output contract." +us,scenario_098,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model incorrectly exempted the elderly household from the gross-income eligibility test and then used medical and uncapped shelter deductions to manufacture net-income eligibility. It also applied the one-person minimum allotment after the benefit formula reached zero, even though the household first fails SNAP income eligibility." +us,scenario_098,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model evaluated only a deduction-reduced net-income test and omitted the basic income eligibility failure produced by approximately $35,035 of annual income. Because the household is ineligible, the one-person minimum monthly allotment does not apply." +us,scenario_098,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model labeled the household “very low-income” without adding Social Security and pension income and accounting for the farm-rent loss, which yields approximately $35,035 annually. That income fails PolicyEngine’s SNAP eligibility threshold, so estimating a positive benefit from age and assets was erroneous." +us,scenario_098,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the large rent and shelter deductions as reducing SNAP net income to zero. The household fails the income eligibility requirement before those deductions can support a benefit, so awarding the full maximum allotment was erroneous." +us,scenario_098,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value or explanation, violating the required structured-output contract." us,scenario_098,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_098,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model applied Illinois’s 4.95% rate to an AGI-based amount after exemptions but failed to subtract the household’s qualifying Social Security and private-pension retirement income. Those Illinois retirement-income subtractions, together with the farm-rent loss, reduce taxable income to zero." -us,scenario_098,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model treated $4,440 as Illinois income remaining before personal and senior exemptions, leaving $515 taxable, but failed to apply Illinois’s subtraction for qualifying pension retirement income. Applying that subtraction eliminates the residual taxable income before the 4.95% rate is used." +us,scenario_098,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model treated $35,035 of net household cash income as Illinois taxable income and merely reduced it by exemptions. It failed to start from federal AGI, which excludes the nontaxable Social Security income, and failed to apply Illinois's subtraction for qualifying pension income, leaving no Illinois taxable base." +us,scenario_098,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model correctly approached a $4,440 federal-AGI starting point but then applied only the personal and senior exemptions. It omitted Illinois's subtraction modification for the taxable private pension included in federal AGI, which reduces Illinois taxable income to zero before exemptions and the 4.95% rate." +us,scenario_098,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model incorrectly included Social Security and the energy subsidy in Illinois base income, failed to incorporate the farm-rent loss correctly, and invented a standard deduction that Illinois does not use. It also omitted Illinois's subtraction for qualifying Social Security and pension income; the correct state taxable base is zero before applying the 4.95% rate." us,scenario_098,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,child1_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,child1_early_head_start_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -5142,102 +5520,108 @@ us,scenario_099,child1_medicaid_eligible,kimi-k3,parse_contract_failure,missing_ us,scenario_099,child1_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,child1_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,child2_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_099,child2_early_head_start_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model substituted a nonexistent 100% of California state median income standard for Early Head Start's 100% of the federal poverty guideline test. It also treated age four as within Early Head Start's ordinary birth-to-three target group; the household's approximately $159,676 of income independently makes child 2 ineligible." -us,scenario_099,child2_early_head_start_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output for child2_early_head_start_eligible, violating the required submission contract. The required value is 0 because household income of approximately $159,676 exceeds the Early Head Start income limit tied to 100% of the federal poverty guideline." -us,scenario_099,child2_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age four as sufficient for Head Start eligibility. Age only places Child 2 in the program's target age group; the household must also satisfy the income test or a categorical pathway, neither of which applies." -us,scenario_099,child2_head_start_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no value for the requested output, violating the required output contract. The required result was 0 because the household exceeds the Head Start income threshold and has no categorical eligibility pathway." +us,scenario_099,child2_early_head_start_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model replaced Early Head Start's federal poverty-guideline test with a nonexistent allowance up to 100% of California median income. It also treated age four as within Early Head Start's birth-to-three eligibility range; the child's employer-sponsored insurance does not establish eligibility. +us,scenario_099,child2_early_head_start_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no output for child2_early_head_start_eligible. The required result is 0 because household income is far above the Early Head Start federal poverty-guideline limit. +us,scenario_099,child2_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 4 as sufficient for Head Start eligibility. Age only establishes the preschool-age condition; the household fails the income test and has no categorical eligibility basis, so Child 2 is not eligible." +us,scenario_099,child2_head_start_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model omitted the required child2_head_start_eligible output. The required result is 0 because the household's approximately $159,676 income exceeds the Head Start income threshold and no categorical eligibility pathway applies." us,scenario_099,child2_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,child2_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_099,child2_wic_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model treated age four as sufficient and disregarded the mandatory 185%-of-poverty income test after acknowledging that household income exceeded its own stated threshold. Its claim that WIC serves families above that limit did not establish an applicable adjunctive-eligibility pathway, and household assets do not replace the failed income test." -us,scenario_099,child2_wic_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model applied only WIC's under-age-five categorical condition and omitted the separate income requirement. Approximately $159,676 in annual household income exceeds 185% of the federal poverty guideline for a family of four." -us,scenario_099,child2_wic_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model converted satisfaction of the WIC age condition directly into eligibility without testing household income. The household's approximately $159,676 annual income is above the applicable 185%-of-poverty ceiling." -us,scenario_099,child2_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child2_wic_eligible, violating the required output contract." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used a $32,600 standard deduction instead of $34,200 and reduced taxable income through unsupported $2,000 charitable and $1,575 auto-loan deductions. It also applied the 20% CDCC floor instead of the applicable 32% rate, understating that credit by $720." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the pretax 401(k) contributions, net capital loss, and applicable above-the-line deductions correctly, producing $136,302 of taxable income instead of $125,385.05. It also allowed only $3,174 of credits instead of the $4,400 CTC plus $1,920 CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a $31,500 standard deduction rather than $34,200, limited the modeled capital-loss reduction to $3,000, and used estimated brackets. It also applied obsolete $2,000-per-child CTC and 20% CDCC amounts instead of $2,200 per child and 32% of $6,000." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's own derivation reaches about $12,421 after credits, but it submitted $18,962, so the final value does not follow its computation. Its derivation also double-counted the $128 qualified dividend, used a $31,500 deduction, and applied obsolete $4,000 CTC and $1,200 CDCC amounts." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model omitted the CDCC entirely even though both children qualify and $6,000 of expenses generates a $1,920 nonrefundable credit. It also used $4,000 rather than $4,400 of CTC and an estimated standard deduction and tax calculation." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model incorrectly treated the refundable CTC ceiling as splitting each child's credit into only $300 nonrefundable and $1,700 refundable despite ample pre-credit tax liability; the full $4,400 CTC offsets tax here. It also used a 20% CDCC rate instead of 32%, a $30,000 standard deduction instead of $34,200, and failed to reproduce the traced deductions." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model computed tax before credits near $19,900 and described subtracting the CTC and CDCC, yet submitted $24,500, an amount exceeding its own pre-credit tax. It also used an estimated $30,300 standard deduction rather than $34,200 and did not apply the $4,400 CTC and $1,920 CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model invented a much lower $134,987 AGI, personal exemptions, and itemized deductions, yielding $88,929 of taxable income instead of $125,385.05. It then used only $750 of CTC and a 20% CDCC rather than the $4,400 CTC and $1,920 CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied a post-TCJA-expiration framework with personal exemptions and itemized deductions instead of the $34,200 standard deduction. It also used a $2,000 total CTC and $1,200 CDCC rather than $4,400 and $1,920." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The stated $164,481 AGI fails to incorporate the traced pretax contributions, investment income, net capital loss, and above-the-line deductions that produce $159,585.05. Its answer also implies that the full $6,320 of nonrefundable credits was not subtracted after calculating tax." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed a TCJA sunset, invented personal exemptions, and reduced AGI to $135,169, producing taxable income far below $125,385.05. It also phaseout-reduced the CTC to $700 even though the household receives a $4,400 nonrefundable CTC, and used a 20% rather than 32% CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used a pre-TCJA regime with personal exemptions and unsupported itemized deductions, reducing taxable income to $84,650 rather than $125,385.05. It also incorrectly limited the CTC to $750 and the CDCC to $1,200." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model supplied no computation beyond naming unspecified credits. The correct sequence is $16,999.75 of tax before credits minus $4,400 of CTC and $1,920 of CDCC, not $11,105." -us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model invented a $15,600 CTC and a $1,404 saver's credit; neither follows from the household inputs or the traced credit calculation. It also used one-child CDCC expenses despite two qualifying children and failed to apply the actual $4,400 CTC and $1,920 CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The answer is only $278 below the traced $16,999.75 tax before credits, showing that it failed to subtract most of the $6,320 in nonrefundable credits. Both children generate a $4,400 CTC, and $6,000 of childcare expenses generates a $1,920 CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model asserted that it applied child-related credits but left $14,696, which fails to subtract the traced $4,400 CTC and $1,920 CDCC from $16,999.75. Its generic reference to education-related credits does not implement the actual credit pathways." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a $32,200 standard deduction instead of $34,200 and calculated $127,566 of taxable income rather than $125,385.05. It also applied a 20% CDCC of $1,200 instead of the applicable 32% credit of $1,920." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used the correct $4,400 CTC but applied only a $1,200 CDCC, missing the 32% rate that produces $1,920. Its estimated taxable-income calculation also failed to reproduce the traced $159,585.05 AGI and $34,200 standard deduction." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model overstated taxable income at $127,528 instead of $125,385.05 and consequently overstated pre-credit tax. It also used a $1,200 CDCC rather than $1,920, failing to apply the 32% rate to the $6,000 expense cap." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used itemized deductions even though the traced computation selects the $34,200 standard deduction. It also used an obsolete $4,000 CTC and 20% CDCC instead of the $4,400 CTC and $1,920 CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model substituted a rough family-level estimate for the required calculation. The traced taxable income produces $16,999.75 before credits, and subtracting $6,320 of specified nonrefundable credits yields $10,679.75, not $14,000." -us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied restored personal exemptions and pre-TCJA brackets instead of the $34,200 standard deduction and current rate schedule. It also treated the CTC as fully phased out even though both children generate the full $4,400 credit, and it used a $1,200 rather than $1,920 CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used pre-TCJA 10%, 15%, and 25% brackets and an unsupported $110,683 ordinary-taxable-income figure rather than the traced $125,385.05 taxable income calculation. It also phaseout-reduced the CTC to $751 and used a $1,200 CDCC instead of $4,400 and $1,920." -us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for the requested variable. -us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for the requested variable. -us,scenario_099,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated charitable and other deductions plus credits as eliminating liability without calculating their limits or comparing itemized deductions with the $34,200 standard deduction. The actual $16,999.75 pre-credit tax exceeds the $6,320 of nonrefundable credits, leaving $10,679.75." -us,scenario_099,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own revised computation ends at about $11,730, but it submitted $24,711.50, so the final number contradicts its stated arithmetic. The derivation also deducted the full $6,478 capital loss directly while omitting the traced above-the-line treatment, and used a $4,000 CTC and $1,200 CDCC instead of $4,400 and $1,920." -us,scenario_099,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model correctly recognized that the ACTC is limited to CTC left unused against tax liability, then contradicted that rule by inventing a $1,200 refundable amount despite its own calculation showing the entire CTC was absorbed as a nonrefundable credit. The 15% earned-income formula is only a cap on refundability and does not make any CTC refundable when no credit remains after offsetting tax." -us,scenario_099,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no federal_refundable_credits value or explanation, violating the required structured-output contract." -us,scenario_099,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no federal_refundable_credits value or explanation, violating the required structured-output contract." -us,scenario_099,free_school_meals_eligible,claude-fable-5,llm_error,categorical_eligibility,False,The model applied the federal 130% FPL income test as dispositive and omitted California's universal free school meals pathway. The universal program supersedes the household's 5.03 FPG ratio and provides free meals to the enrolled K-12 child. -us,scenario_099,free_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly restricted eligibility to the 130% FPL test and treated the household's assets as further grounds for denial. California provides universal free school meals to enrolled students without an income or asset test. -us,scenario_099,free_school_meals_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model treated the federal income thresholds as controlling and omitted California's universal free school meals program. Income above 185% FPL does not disqualify the enrolled K-12 child from California's universal program. -us,scenario_099,free_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model applied only the ordinary 130% FPL free-meal test and concluded that high income eliminated support. California's universal program supplies positive free-meal support regardless of income. -us,scenario_099,free_school_meals_eligible,claude-opus-5,llm_error,categorical_eligibility,False,The model required either income below 130% FPL or categorical eligibility and omitted the independent California universal-meals pathway. Neither high income nor the absence of categorical eligibility blocks free meals under that pathway. -us,scenario_099,free_school_meals_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model treated the National School Lunch Program's ordinary 130% FPL limit as the sole eligibility rule. California's universal free school meals program covers the enrolled child despite household income above that limit. -us,scenario_099,free_school_meals_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model relied exclusively on the federal 130% and 185% poverty thresholds. California's universal program overrides those income-based tiers and places the enrolled child in the FREE tier. -us,scenario_099,free_school_meals_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,The answer uses income above 130% FPL as an automatic disqualification. California universal free school meals make the enrolled K-12 child eligible irrespective of the 5.03 FPG ratio. -us,scenario_099,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model treated excess household income as dispositive and omitted California's universal free school meals rule. That rule produces positive free-meal support for the enrolled child without an income test. -us,scenario_099,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model applied the 130% FPL test as the only eligibility route. California's universal program covers enrolled students even when household income exceeds 130% FPL. -us,scenario_099,free_school_meals_eligible,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model denied eligibility solely because income exceeded the ordinary threshold. It omitted California's universal free school meals program, which makes the enrolled child eligible regardless of income." -us,scenario_099,free_school_meals_eligible,glm-5.2,llm_error,categorical_eligibility,False,The model compared its estimated adjusted gross income with 130% FPL and stopped there. California's universal free school meals pathway does not depend on adjusted gross income and covers the enrolled K-12 child. -us,scenario_099,free_school_meals_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model inferred no benefit from the household's high income and omitted California's universal free school meals program. The state program generates positive annual support for the enrolled child regardless of the income thresholds. -us,scenario_099,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model treated the absence of an explicit eligibility indicator and the household's wages as grounds for denial. The stated California residence activates the state's universal free school meals rule for the enrolled nine-year-old. -us,scenario_099,free_school_meals_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model required satisfaction of the 130% FPL limit or a categorical eligibility route. It omitted California's separate universal-meals pathway, under which neither condition is required." -us,scenario_099,free_school_meals_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model treated income above the ordinary free-meal threshold as conclusive. California's universal program supersedes that threshold for enrolled students and yields positive free-meal support. -us,scenario_099,free_school_meals_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model denied free meals based solely on high household income. It failed to apply California's universal free school meals program, which has no household-income cutoff." -us,scenario_099,free_school_meals_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model used high income as an automatic bar to free school meals. California universally covers enrolled students, so the household's 5.03 FPG ratio does not prevent eligibility." -us,scenario_099,free_school_meals_eligible,grok-4.5,llm_error,categorical_eligibility,False,The model considered only the 130% FPL and categorical-eligibility routes and therefore predicted zero support. California's universal program is an independent route that places the enrolled child in the FREE tier. -us,scenario_099,free_school_meals_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,The model incorrectly made the 130% FPL limit dispositive. California's universal free school meals program covers the enrolled K-12 child even though household income exceeds that limit. -us,scenario_099,free_school_meals_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model checked household income and categorical factors such as SNAP, TANF, and foster status but omitted California's universal-meals pathway. That pathway requires neither low income nor categorical eligibility and generates positive annual support." -us,scenario_099,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for free_school_meals_eligible, violating the required output contract." -us,scenario_099,free_school_meals_eligible,minimax-m3,llm_error,categorical_eligibility,False,The model treated income above the ordinary program limits as disqualifying. California's universal free school meals program covers the enrolled child without applying those limits. -us,scenario_099,free_school_meals_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model applied the 130% FPL threshold as the sole route to free meals. It omitted California's universal program, which makes the enrolled K-12 child eligible regardless of the household's approximately $165,000 in wages." +us,scenario_099,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model recognized that household income exceeded the WIC threshold but incorrectly treated age eligibility as sufficient and invented an exception for working families above 185% of the federal poverty level. WIC requires financial eligibility in addition to being under age five, and the household's income fails that test; the listed assets do not cure the income failure." +us,scenario_099,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the under-five age requirement as the complete WIC eligibility test. Child 2 satisfies the categorical age rule, but household income of approximately $159,676 exceeds the 185%-of-poverty income limit, making the child ineligible." +us,scenario_099,child2_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model inferred eligibility solely from child 2's age and never applied WIC's household-income test. The household's approximately $159,676 annual income is above the applicable 185%-of-poverty limit, so the categorical age condition does not produce eligibility." +us,scenario_099,child2_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output for child2_wic_eligible, violating the required submission contract. It therefore never reported the result produced by applying the WIC income limit: value 0." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model applied the CDCC's old 20% floor instead of the applicable 32% rate, understating that credit by $720. It also used estimated deductions and brackets rather than the $34,200 standard deduction and exact 2026 rate thresholds." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model failed to reduce income for the pre-tax 401(k) contributions, net capital-loss deduction, and other above-the-line deductions, then used a $29,200 standard deduction instead of $34,200. It also replaced the actual $6,320 of nonrefundable credits with an unsupported $3,174." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used estimated 2026 brackets and a $31,500 standard deduction instead of the exact $34,200 deduction. It also applied a $4,000 CTC and a 20% CDCC, whereas the applicable nonrefundable credits are $4,400 of CTC and $1,920 of CDCC." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's own calculation reaches about $12,421 after credits, but it submitted $18,962, so its final value does not follow from its stated arithmetic. Its calculation also double-counted the $128 qualified dividend and used obsolete $4,000 CTC and 20% CDCC amounts." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model effectively omitted the CDCC despite $6,000 of eligible expenses generating a $1,920 credit at the applicable 32% rate. It also used a $4,000 CTC rather than $4,400 and estimated the standard deduction and tax brackets." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model incorrectly treated the refundable ceiling as dividing each child's CTC into only $300 nonrefundable and $1,700 refundable even though the full $4,400 CTC is usable against the household's pre-credit tax. It also applied the old 20% CDCC rate instead of 32% and used a $30,000 standard deduction instead of $34,200." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The submitted $24,500 contradicts the model's own pre-credit tax estimate of about $19,900 and cannot result from subtracting the CTC and CDCC it identified. It also invoked AMT and investment-tax adjustments that are not part of this output and do not support an increase above regular tax before credits." +us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model invented a $134,987 AGI, personal exemptions, and itemized deductions instead of using $159,585.05 of AGI and the $34,200 standard deduction. It also reduced the CTC to $750 and the CDCC to $1,200 rather than applying $4,400 and $1,920." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied a post-TCJA-expiration regime with personal exemptions and itemized deductions instead of the $34,200 standard deduction. It also used only $2,000 of CTC and a 20% CDCC rather than $4,400 and a 32% CDCC." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model overstated AGI at $164,481 by failing to incorporate the full pre-tax 401(k), capital-loss, and above-the-line adjustments that produce $159,585.05. Its answer also implies that the $6,320 of nonrefundable credits was not fully subtracted." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly used a TCJA-sunset calculation with a $135,169 AGI, personal exemptions, and itemized deductions. The applicable computation uses $159,585.05 of AGI, a $34,200 standard deduction, and full nonrefundable credits of $6,320." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied pre-TCJA rules, including personal exemptions and invented itemized deductions, producing taxable income far below $125,385.05. It also reduced the CTC to $750 and used a $1,200 CDCC instead of the applicable $4,400 and $1,920." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model supplied no derivation for its $11,105 figure. The exact computation is $16,999.75 of tax before credits minus $6,320 of nonrefundable credits, not the $5,894.75 reduction implied by its answer." +us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model invented a $15,600 CTC and a $1,404 saver's credit; the applicable credits are a $4,400 CTC and a $1,920 CDCC, with no saver's credit in the trace. It also mishandled the capital-loss limitation and used inconsistent standard-deduction amounts." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The answer implies that almost none of the $6,320 in nonrefundable credits was deducted from the $16,999.75 pre-credit tax. Both qualifying children generate $4,400 of usable CTC, and the childcare expenses generate a $1,920 CDCC." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The answer implies only $2,303.75 was subtracted from the $16,999.75 pre-credit tax instead of the full $6,320 in nonrefundable credits. The computation requires $4,400 of CTC plus a $1,920 CDCC." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a $32,200 standard deduction instead of $34,200, overstating taxable income and regular tax. It also used a 20% CDCC of $1,200 instead of the applicable 32% credit of $1,920." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Although the model identified the $4,400 CTC, it used a $1,200 CDCC rather than $1,920 because it applied the wrong credit percentage. Its implied pre-credit tax also differs from the exact $16,999.75 derived from $125,385.05 of taxable income." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model used taxable income of $127,528 instead of $125,385.05 and consequently overstated pre-credit tax. It then applied a 20% CDCC of $1,200 rather than the applicable 32% credit of $1,920." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model used a $4,000 CTC instead of $4,400 and a $1,200 CDCC instead of $1,920. It also described itemized deductions as controlling even though the derivation uses the $34,200 standard deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model used a family-level rule-of-thumb rather than computing the requested tax measure. The exact path yields $16,999.75 before credits and subtracts $6,320 of nonrefundable credits, not the roughly $3,000 reduction embedded in its estimate." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied restored pre-TCJA brackets, personal exemptions, and itemized deductions instead of the $34,200 standard deduction and applicable 2026 brackets. It also treated the CTC as fully phased out even though the household receives a $4,400 nonrefundable CTC." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used pre-TCJA 10%/15%/25% brackets and an erroneous $134,987 AGI rather than the applicable $159,585.05 AGI and current brackets. It also phaseout-reduced the CTC to $751 and used a 20% CDCC instead of $4,400 of CTC and a $1,920 CDCC." +us,scenario_099,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model used a $32,200 standard deduction rather than $34,200, which overstated taxable income and pre-credit tax. It also calculated the CDCC at 20% of eligible expenses instead of the applicable 32%, understating the credit by $720." +us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric value or explanation for the requested output. +us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model returned no numeric value or explanation for the requested output. +us,scenario_099,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model set the liability to zero despite acknowledging substantial tax before credits. The household has $16,999.75 of pre-credit tax and only $6,320 of usable nonrefundable credits, leaving $10,679.75 rather than eliminating the liability." +us,scenario_099,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The submitted $24,711.50 contradicts the model's own successive calculations of $11,804.36 and $11,730 after credits. Its underlying work also deducted the full $6,478 capital loss instead of the $3,000 annual limit and used a $4,000 CTC and $1,200 CDCC instead of $4,400 and $1,920." +us,scenario_099,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model overstated AGI at $164,894 and taxable income at $134,894 by miscomputing both retirement adjustments and the standard deduction. It also omitted the $4,400 CTC and treated the CDCC as $1,200 rather than $1,920." +us,scenario_099,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated the ACTC earned-income formula as creating a refundable payment even though the household's tax liability fully absorbs the CTC as a nonrefundable credit. Its asserted $1,200 is unsupported by its own calculation: no unused CTC remains to become refundable, so refundable CTC is $0." +us,scenario_099,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output for federal_refundable_credits, violating the required submit_outputs contract and leaving the requested value missing." +us,scenario_099,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output for federal_refundable_credits, violating the required submit_outputs contract and leaving the requested value missing." +us,scenario_099,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model correctly stated that the dependent credits were nonrefundable and fully absorbed by tax liability, then incorrectly reported their $600 amount as federal_refundable_credits. A nonrefundable credit cannot enter this output, and its own reasoning therefore yields $0 rather than $600." +us,scenario_099,free_school_meals_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the federal 130% poverty limit as the exclusive free-meal pathway. It omitted California's universal free school meals program, which qualifies the enrolled nine-year-old despite the household's 5.03 FPG ratio." +us,scenario_099,free_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied an income test and introduced an asset test, neither of which bars meals under California's universal program. The enrolled nine-year-old receives free meals regardless of the household's income or stock and bank assets." +us,scenario_099,free_school_meals_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model incorrectly used the 185% FPL reduced-price threshold to deny free meals and omitted California's universal pathway. State universal eligibility sets the tier to FREE regardless of the household's 5.03 FPG ratio. +us,scenario_099,free_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated the ordinary 130% FPL income rule as controlling. California's universal program supersedes that income test and provides positive free-meal support for the enrolled nine-year-old. +us,scenario_099,free_school_meals_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model considered only income eligibility and categorical eligibility through programs such as SNAP or TANF. It omitted California's separate universal free-meal pathway, which applies even though both of those ordinary pathways fail." +us,scenario_099,free_school_meals_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model applied the National School Lunch Program's ordinary 130% FPL test as an exclusive condition. California universally provides free school meals, so the enrolled nine-year-old qualifies despite household income above that threshold." +us,scenario_099,free_school_meals_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model stopped after comparing income with the federal free and reduced-price thresholds. It failed to apply California's universal free school meals program, which sets the household's school-meal tier to FREE regardless of income." +us,scenario_099,free_school_meals_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,The model treated income above 130% FPL as dispositive. California's universal program bypasses that threshold and supplies free meals to the enrolled nine-year-old. +us,scenario_099,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model relied solely on the ordinary income threshold. It omitted California's universal free-meal rule, under which income does not prevent the enrolled student from qualifying." +us,scenario_099,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model correctly recognized that income exceeds 130% FPL but incorrectly treated that fact as ending the eligibility analysis. California's universal program independently qualifies the enrolled nine-year-old for free meals. +us,scenario_099,free_school_meals_eligible,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model applied only an income-threshold pathway. California's universal free school meals program supersedes the failed income test and yields positive annual support. +us,scenario_099,free_school_meals_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model compared its adjusted-income estimate with 130% FPL and stopped. The decisive rule is California's universal free-meal program, which qualifies the enrolled student without regard to adjusted income." +us,scenario_099,free_school_meals_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model inferred no benefit solely from high household income. It omitted California's universal free school meals pathway, which produces positive support for the enrolled nine-year-old at any income." +us,scenario_099,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an expressly listed eligibility fact and the high wages as grounds for denial. The listed California residence activates the state's universal program, and the nine-year-old's K–12 enrollment yields free-meal support." +us,scenario_099,free_school_meals_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model limited eligibility to the 130% FPL test and categorical participation such as SNAP or TANF. California universal free meals form a third, controlling pathway that qualifies the enrolled student even when the first two fail." +us,scenario_099,free_school_meals_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model treated household income above the ordinary threshold as disqualifying. California's universal program overrides that threshold and returns positive free-meal support. +us,scenario_099,free_school_meals_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model used the income threshold as the sole eligibility rule. It missed California's universal free school meals program, which applies regardless of the household's 5.03 FPG ratio." +us,scenario_099,free_school_meals_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly made high income dispositive. California provides universal free school meals, so high income does not disqualify the enrolled nine-year-old." +us,scenario_099,free_school_meals_eligible,grok-4.5,llm_error,categorical_eligibility,False,The model checked the 130% FPL and categorical-eligibility pathways but omitted California's universal pathway. That state rule sets the tier to FREE and produces positive support despite high income and no categorical eligibility. +us,scenario_099,free_school_meals_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,The model treated the federal 130% FPL limit as mandatory for this California household. California's universal program removes that income condition for the enrolled student. +us,scenario_099,free_school_meals_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model checked income and categorical factors such as SNAP, TANF, and foster care but omitted the independent state-universal pathway. California's universal program qualifies the enrolled nine-year-old even though income and categorical eligibility do not." +us,scenario_099,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied neither a numeric value nor an explanation for the requested output, violating the submission contract." +us,scenario_099,free_school_meals_eligible,minimax-m3,llm_error,categorical_eligibility,False,The model denied eligibility based only on federal income limits. California's universal free school meals program applies without an income limit and yields positive annual support. +us,scenario_099,free_school_meals_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated income above 130% FPL as a complete bar. It failed to apply California's universal program, which qualifies the enrolled nine-year-old regardless of the household's income." +us,scenario_099,free_school_meals_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The model relied exclusively on the ordinary income threshold. California universal free meals supersede that test and produce positive support for the enrolled student. us,scenario_099,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_099,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model applied a 1.2% California SDI rate instead of 1.3%, then submitted $14,118 despite its own components summing to $14,602.50. Correct SDI is $2,145, producing total payroll tax of $14,767.50." -us,scenario_099,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly treated California as having no mandatory employee payroll tax and omitted the $2,145 employee SDI contribution. Federal FICA of $12,622.50 plus California SDI equals $14,767.50." -us,scenario_099,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model used a 1.2% California SDI rate instead of 1.3%, and its submitted $13,197.50 does not equal any total it calculated. The correct SDI component is $2,145, which raises $12,622.50 of FICA to $14,767.50." -us,scenario_099,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model explicitly excluded California SDI after calculating it at the wrong 1.1% rate. The requested output includes mandatory state payroll taxes, so $2,145 of California SDI must be added to $12,622.50 of federal FICA." -us,scenario_099,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model calculated only federal Social Security and Medicare taxes, rounded Medicare down by $0.50, and omitted $2,145 of California SDI. The correct components total $14,767.50." -us,scenario_099,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,The model included California SDI but applied a 1.1% rate instead of 1.3%. This understated SDI by $330 and payroll tax by the same amount. -us,scenario_099,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model vacillated between excluding California SDI and including it at 1.1%, then submitted a value matching neither derivation. California SDI is mandatory and equals 1.3% of $165,000, or $2,145, making the total $14,767.50." -us,scenario_099,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted all listed employer-sponsored insurance premiums from FICA and SDI wages without a basis in the benchmark inputs, reducing the payroll-tax base from $165,000 to $140,403. It also used 1.1% rather than 1.3% for California SDI." -us,scenario_099,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a 1.1% California SDI rate instead of 1.3%. Correct SDI is $2,145 rather than $1,815, so the total is $14,767.50." -us,scenario_099,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model stopped after computing $12,622.50 of federal Social Security and Medicare taxes. It omitted the mandatory $2,145 California SDI contribution included in the requested payroll-tax output." -us,scenario_099,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly subtracted $24,597 of employer-sponsored insurance premiums from the stated gross-wage payroll-tax base. It also applied 1.1% instead of 1.3% for California SDI; the correct calculation uses all $165,000 of wages." -us,scenario_099,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a 1.1% California SDI rate instead of 1.3%. That reduced the state contribution from $2,145 to $1,815 and understated the total by $330." -us,scenario_099,payroll_tax,gemini-3.6-flash,llm_error,other,False,"The submitted total does not follow from the required components: Social Security and Medicare alone equal $12,622.50, and California SDI adds $2,145. The model failed to compute those components to the required $14,767.50 total." -us,scenario_099,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly treated the household's combined wages as subject to one Social Security wage cap, even though the cap applies separately to each worker and neither worker reaches it. It also omitted $2,145 of California SDI and submitted a number inconsistent with its intermediate calculations." -us,scenario_099,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model calculated federal FICA only and omitted the mandatory $2,145 California SDI contribution. It also rounded the $2,392.50 Medicare component down by $0.50." -us,scenario_099,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's claimed federal FICA total of $13,577 is arithmetically wrong: 7.65% of $165,000 is $12,622.50. It also omitted the $2,145 California SDI component." -us,scenario_099,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"The model applied a 1.2% California SDI rate instead of 1.3%. This understated SDI by $165 and produced $14,602.50 instead of $14,767.50." -us,scenario_099,payroll_tax,gpt-5.6-luna,llm_error,other,False,"The model's submitted $14,312.40 does not equal federal FICA plus its stated approximate 1.2% California SDI component. California SDI must be calculated exactly at 1.3%, adding $2,145 to $12,622.50 of FICA." -us,scenario_099,payroll_tax,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model estimated California SDI at $1,980 using 1.2% rather than the applicable 1.3%. Correct SDI is $2,145, increasing the total by $165 to $14,767.50." -us,scenario_099,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used an approximate 1.2% California SDI calculation instead of the exact 1.3% rate and rounded Medicare to a whole dollar. Exact components are $10,230, $2,392.50, and $2,145, totaling $14,767.50." -us,scenario_099,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The answer is a rounded federal-FICA-only calculation and omits mandatory California SDI. The omitted state payroll tax is $2,145, which must be added to exact FICA of $12,622.50." -us,scenario_099,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"The model included California SDI at 1.1% instead of 1.3% and rounded its resulting total. Correct SDI is $2,145, so exact payroll tax is $14,767.50." -us,scenario_099,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages by the full $24,597 of listed employer-sponsored insurance premiums and omitted California SDI entirely. The benchmark uses $165,000 of gross wages for FICA and adds $2,145 of mandatory state SDI." -us,scenario_099,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, violating the required output contract." -us,scenario_099,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, violating the required output contract." -us,scenario_099,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly listed per-worker federal FICA amounts of $6,502.50 and $6,120 but then submitted roughly twice their $12,622.50 sum. It also omitted $2,145 of California SDI, which produces the correct total of $14,767.50." -us,scenario_099,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model recognized that mandatory California SDI belongs in the output but discarded it in the submitted value, leaving only federal FICA. It also used 1.1% in its abandoned SDI calculation instead of 1.3%; the required state component is $2,145." +us,scenario_099,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model used a 1.2% California SDI rate instead of 1.3%, then submitted $14,118 even though its own components summed to $14,602.50. Applying $2,145 of SDI to $12,622.50 of FICA yields $14,767.50." +us,scenario_099,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model excluded California SDI after incorrectly asserting that California has no mandatory employee payroll tax. The required $2,145 employee SDI contribution raises federal FICA of $12,622.50 to $14,767.50." +us,scenario_099,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model used a 1.2% California SDI rate instead of 1.3%, and its submitted $13,197.50 also contradicted its own recomputed total of $14,602.50. The correct SDI amount is $2,145, producing total payroll tax of $14,767.50." +us,scenario_099,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model ultimately omitted mandatory California SDI by limiting the output to federal FICA, despite identifying SDI as applicable. California SDI is $2,145 at 1.3%, not $1,815 at 1.1%, and must be added to $12,622.50." +us,scenario_099,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model omitted the mandatory $2,145 California SDI contribution and also rounded Medicare tax down by $0.50. Social Security of $10,230, Medicare of $2,392.50, and SDI of $2,145 total $14,767.50." +us,scenario_099,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model applied a 1.1% California SDI rate instead of the 1.3% rate used for 2026. This understated SDI by $330, yielding $14,437.50 instead of $14,767.50." +us,scenario_099,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model vacillated between excluding SDI and applying an incorrect 1.1% rate, then submitted $12,903.50 without support from either calculation. Mandatory California SDI is $2,145, which added to $12,622.50 of FICA yields $14,767.50." +us,scenario_099,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the listed employer-sponsored insurance premiums from wages when constructing the Social Security, Medicare, and SDI bases. The trace applies all three taxes to the full $165,000 of gross wages and uses a 1.3% California SDI rate." +us,scenario_099,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model used a 1.1% California SDI rate instead of 1.3%. Correct SDI is $2,145 rather than $1,815, increasing the total by $330 to $14,767.50." +us,scenario_099,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model calculated only employee Social Security and Medicare tax and omitted mandatory California SDI. Adding the $2,145 SDI contribution to $12,622.50 of federal FICA gives $14,767.50." +us,scenario_099,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly deducted $24,597 of employer-sponsored insurance premiums from the payroll-tax wage base and used a 1.1% SDI rate. PolicyEngine applies FICA and 1.3% California SDI to the full $165,000 of listed gross wages." +us,scenario_099,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model applied California SDI at 1.1% rather than 1.3%. That understated the state payroll tax by $330 and produced $14,437.50 instead of $14,767.50." +us,scenario_099,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The submitted total does not reflect the required components: federal Social Security and Medicare already equal $12,622.50, and California SDI adds $2,145. The correct component sum is $14,767.50." +us,scenario_099,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model misapplied the Social Security wage cap to combined household wages even though the cap applies separately to each worker, and it omitted California SDI. Each spouse earns below the cap, so Social Security is 6.2% of the full $165,000, with $2,392.50 of Medicare and $2,145 of SDI added." +us,scenario_099,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model restricted payroll tax to federal FICA and omitted mandatory California SDI, while also rounding away $0.50 of Medicare tax. The correct calculation adds $2,145 of SDI to $12,622.50 of Social Security and Medicare tax." +us,scenario_099,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's $13,577 total does not equal employee FICA on $165,000 and omits most of the mandatory California SDI liability. Federal FICA is $12,622.50 and California SDI is $2,145, totaling $14,767.50." +us,scenario_099,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model applied California SDI at 1.2% instead of 1.3%. Correct SDI is $2,145 rather than $1,980, so payroll tax is $165 higher at $14,767.50." +us,scenario_099,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The submitted amount neither implements the stated approximate 1.2% SDI calculation nor the required 1.3% rate. California SDI is $2,145, and adding it to $12,622.50 of FICA yields $14,767.50." +us,scenario_099,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,The model used a 1.2% California SDI rate instead of 1.3%. This understated SDI by $165 and payroll tax by the same amount. +us,scenario_099,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model used an approximate 1.2% California SDI amount instead of $2,145 at 1.3%, and rounded Medicare tax. Exact Social Security, Medicare, and SDI liabilities sum to $14,767.50." +us,scenario_099,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model approximated federal FICA and omitted mandatory California SDI. Exact federal FICA is $12,622.50, and the additional $2,145 SDI contribution produces $14,767.50." +us,scenario_099,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model applied California SDI at 1.1% rather than 1.3% and rounded the result. Correct SDI is $2,145, bringing total employee payroll tax to $14,767.50." +us,scenario_099,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly subtracted employer-sponsored insurance premiums from gross wages and excluded California SDI entirely. The full $165,000 is the traced payroll-tax base, producing $12,622.50 of FICA plus $2,145 of SDI." +us,scenario_099,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model applied California SDI at 1.2% instead of 1.3% and rounded its result. SDI is $2,145 rather than $1,980, so the exact total is $14,767.50." +us,scenario_099,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, violating the required output contract." +us,scenario_099,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, violating the required output contract." +us,scenario_099,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly calculated employee FICA of $12,622.50 in its reasoning but submitted approximately twice that amount, effectively double-counting FICA, and omitted California SDI. The correct total adds one $2,145 SDI liability to one $12,622.50 FICA liability." +us,scenario_099,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model recognized that mandatory California SDI belongs in the output but discarded it in the submitted value, and its alternative SDI calculation used 1.1% rather than 1.3%. Adding the correct $2,145 SDI amount to $12,622.50 gives $14,767.50." +us,scenario_099,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model calculated California SDI on only part of the head's wages, omitted the spouse's SDI, and produced $472.20 despite stating a 1.2% rate. SDI applies at 1.3% to both workers' full $165,000 of wages, producing $2,145." us,scenario_099,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -5247,1402 +5631,1496 @@ us,scenario_099,spouse_medicaid_eligible,kimi-k3,parse_contract_failure,missing_ us,scenario_099,spouse_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,spouse_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,ssi,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model derived taxable income near the correct amount but overstated the bracket tax, first describing $6,940 and then submitting $6,222 without reconciling its own exemption credits. Tax on $139,108.47 is $5,778.21, and subtracting $1,284.46 yields $4,493.74." -us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $4,754 standard deduction instead of the allowable $20,476.57 itemized deduction and treated unreimbursed employee expenses as an above-the-line reduction. This inflated taxable income from $139,108.47 to $155,605 and also replaced California exemption credits with an invented $146 child credit." -us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model reached taxable income close to the traced value but never completed a coherent California bracket calculation, cycling through incompatible bases and ultimately submitting $5,959. The correct bracket tax is $5,778.21 and the applicable nonrefundable exemption credits are $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model failed to reduce the listed gross wages by the $3,088 of traditional 401(k) deferrals, producing California AGI of $163,010 instead of $159,585.05. It then overstated the bracket tax on its inflated taxable income before subtracting exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied roughly $7,700 of California tax to taxable income near $138,000, far above the $5,778.21 generated by the 2026 joint brackets. It also described the four exemption credits inconsistently and failed to subtract the correct $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used estimated inflation adjustments to old California brackets and assumed about $450 of credit for every household member. The applicable brackets produce $5,778.21, and the actual personal and dependent exemption credits total $1,284.46 rather than $1,800." -us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model reduced the $20,476.57 allowable itemized deduction to an unsupported $18,000–$20,000 range and asserted taxable income near $150,000 instead of $139,108.47. It then failed to subtract the full $1,284.46 of exemption credits from the bracket tax." -us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The submitted $112,436 taxable income omits $26,672.47 of income that remains taxable after California adjustments and deductions. California taxable income is $139,108.47, producing $5,778.21 before the $1,284.46 exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model subtracted only about $1,180 of nonrefundable credits instead of the $1,284.46 personal and dependent exemption credits. Its implied pre-credit tax also differs from the $5,778.21 produced by the 2026 joint brackets." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The $7,845 answer is consistent with applying an overstated California rate or single-filer-style thresholds to the household's income. Joint-filer tax on $139,108.47 is $5,778.21 before $1,284.46 of exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated California AGI by $24,416.05 and overstated itemized deductions by $5,661.43, reducing taxable income to $109,031 instead of $139,108.47. It also subtracted only about $600 rather than $1,284.46 of exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The $2,570 result implies a substantial understatement of bracket tax or taxable income after deductions. The required computation taxes $139,108.47 to $5,778.21 and then subtracts $1,284.46 of exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The unexplained $4,128 answer understates the net liability by $365.74. California's joint brackets produce $5,778.21 on $139,108.47, and the traced nonrefundable credits reduce that amount only to $4,493.74." -us,scenario_099,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model wrongly claimed charitable deductions apply only to the amount exceeding the standard deduction; itemizing instead allows the full applicable charitable and real-estate-tax deductions, totaling $20,476.57. It also treated the $338 educator expense as a tax credit and submitted a number inconsistent with its stated $5,559 calculation." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The unexplained $6,713 answer exceeds even the correct $5,778.21 tax before nonrefundable credits. It therefore failed to apply the 2026 joint brackets correctly and did not subtract the $1,284.46 exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model's unsupported $4,931 result overstates the final liability by $437.26. The bracket calculation yields $5,778.21 and the full nonrefundable exemption-credit subtraction is $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly excluded health premiums and other amounts from California AGI, reducing it to $134,323, and then reported taxable income of $101,811. The traced values are $159,585.05 of AGI, $20,476.57 of deductions, and $139,108.47 of taxable income; its $6,882 tax is also arithmetically incompatible with its stated taxable income." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model said it used the standard deduction, but this household itemizes $20,476.57, exceeding the $11,412 standard deduction. Using the itemized deduction produces $139,108.47 of taxable income and $4,493.74 after exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model identified the principal income, itemized deductions, and exemption credits but converted them to $4,382 using incorrect bracket or credit parameters. The exact stages are $5,778.21 of bracket tax and $1,284.46 of nonrefundable credits." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The unsupported $4,250 estimate understates the result after California deductions and credits. Tax on $139,108.47 is $5,778.21, and subtracting $1,284.46—not a larger estimated credit—produces $4,493.74." -us,scenario_099,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model rounded a rough tax-on-gross-income estimate to $4,500 without computing California AGI, itemized deductions, brackets, and exemption credits. Those stages yield $139,108.47 of taxable income, $5,778.21 before credits, and $4,493.74 after credits." -us,scenario_099,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model came close on AGI and deductions but used estimated bracket boundaries and rounded both tax and credits. The exact bracket tax is $5,778.21 and the exemption credits are $1,284.46, yielding $4,493.74 rather than $4,600." -us,scenario_099,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated taxable income by $28,297.47, using $110,811 instead of $139,108.47. It also asserted that no nonrefundable credits apply even though the personal and two-dependent exemption credits total $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric value or explanation for the requested output, so the required output contract was not satisfied." -us,scenario_099,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no numeric value or explanation for the requested output, so the required output contract was not satisfied." -us,scenario_099,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model set the liability to zero despite $139,108.47 of California taxable income. The joint brackets generate $5,778.21, and the $1,284.46 nonrefundable credits do not eliminate that tax." -us,scenario_099,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model treated the $4,301.50 bracket estimate as the final result and explicitly omitted all nonrefundable credits. California bracket tax is $5,778.21, and personal and dependent exemption credits of $1,284.46 apply; it also used gross-income arithmetic rather than the traced $159,585.05 California AGI." +us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model reached taxable income close to $139,108.47 but overstated the bracket tax, using about $6,940 instead of $5,778.21. Its final $6,222 also does not follow from its own stated $6,940 minus $1,256 computation." +us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $4,754 married-joint standard deduction even though California itemized deductions of $20,476.57 apply. It also treated an educator expense as a tax reduction and failed to apply the $1,284.46 exemption credits correctly." +us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model's bracket computation was internally inconsistent and never correctly calculated $5,778.21 of tax on $139,108.47. It then introduced an unsupported adjustment involving SDI instead of subtracting exactly $1,284.46 of nonrefundable exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the two $1,544 traditional 401(k) contributions from wages, inflating California AGI by $3,088. It then overstated the bracket tax on its inflated taxable income before applying exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied the California brackets incorrectly: tax on roughly $139,108 of joint taxable income is $5,778.21, not about $7,700. It also garbled the exemption-credit arithmetic, describing two adult and two dependent credits as only about $450 instead of $1,284.46." +us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model overstated California exemption credits as roughly $1,800 by assigning the dependent-credit amount to all four household members. The applicable nonrefundable exemption credits total $1,284.46, so $5,778.21 falls to $4,493.74 rather than $4,189." +us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model substituted taxable income of about $150,000 for the traced $139,108.47 despite acknowledging itemized deductions near $20,000. It also failed to separately compute $5,778.21 under the brackets and subtract $1,284.46 of exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to $112,436 instead of $139,108.47, reflecting about $26,672 of excess reductions beyond the applicable $20,476.57 itemized deduction. That understated both the bracket tax and the final liability." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The answer used approximately $1,180 of personal and dependent credits instead of the traced $1,284.46. It also failed to anchor the preceding bracket tax at $5,778.21, yielding $4,665 rather than $4,493.74." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $7,845 result exceeds even the correct $5,778.21 tax before nonrefundable credits. It therefore did not apply the 2026 California joint brackets and the subsequent $1,284.46 exemption-credit subtraction." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated California AGI as $135,169 and overstated deductions as $26,138, producing taxable income of $109,031 instead of $139,108.47. It also used only about $600 of exemption credits rather than $1,284.46." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The $2,570 answer implies a substantial understatement of the bracket tax or an excessive credit subtraction. The correct derivation taxes $139,108.47 to $5,778.21 and subtracts only $1,284.46 of nonrefundable credits." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The unexplained $4,128 result understates the traced bracket-tax-minus-credit calculation by $365.74. The required computation is $5,778.21 minus $1,284.46, with only rounding at the cent level." +us,scenario_099,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model wrongly used a $6,566 standard deduction and claimed the $20,476.57 of itemized deductions did not apply. It also treated the $338 educator expense as a dollar-for-dollar credit, and its submitted $3,202.96 does not match its own stated $5,559 result." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The $6,713 answer exceeds the correct $5,778.21 tax before credits, so it did not correctly apply the California joint brackets. It also omitted the required $1,284.46 nonrefundable exemption-credit subtraction." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model asserted a final amount without showing the bracket or credit calculation. Its $4,931 is $437.26 above the result of taxing $139,108.47 to $5,778.21 and subtracting $1,284.46." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly reduced AGI with health-premium exclusions and arrived at mutually inconsistent taxable-income figures of about $101,811 and $134,323 minus $24,012. Even with its understated taxable income, it produced an impossible $6,882 tax rather than applying the California brackets to $139,108.47 and subtracting exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model said it used the standard deduction, but the household instead itemizes $20,476.57. It consequently failed to reproduce taxable income of $139,108.47 and the resulting $4,493.74 after exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model identified the main AGI, charity, real-estate-tax, and exemption-credit components but misapplied their exact amounts or the 2026 brackets. Those inputs produce $5,778.21 before credits and $4,493.74 after $1,284.46 of credits, not $4,382." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model supplied only a rounded estimate and did not execute the bracket-and-credit calculation. Tax of $5,778.21 less $1,284.46 in exemption credits equals $4,493.74, not $4,250." +us,scenario_099,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model rounded from gross household income without calculating California AGI, itemized deductions, bracket tax, and exemption credits. The full computation yields $4,493.74, so the submitted $4,500 is not a valid cent-level derivation." +us,scenario_099,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model came close on taxable income but overstated bracket tax at about $5,890 instead of $5,778.21 and used about $1,290 rather than $1,284.46 of credits. Those errors produce $4,600 instead of $4,493.74." +us,scenario_099,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used taxable income of $110,811 rather than $139,108.47, reflecting excessive deductions or exclusions. It also incorrectly asserted that no nonrefundable credits apply, omitting $1,284.46 of California exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model's taxable-income estimate was close, but it subtracted only $576 of exemption credits instead of $1,284.46. It also overstated the bracket tax at about $5,866 rather than $5,778.21." +us,scenario_099,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric value or explanation for the requested output. +us,scenario_099,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model returned no numeric value or explanation for the requested output. +us,scenario_099,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model set liability to zero despite $139,108.47 of California taxable income producing $5,778.21 of bracket tax. The $1,284.46 of nonrefundable credits reduce that tax only to $4,493.74." +us,scenario_099,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model treated $4,301.50 as the bracket tax and then asserted that no nonrefundable credits apply. California exemption credits of $1,284.46 apply, and the correct pre-credit bracket tax is $5,778.21; it also misstated the educator-expense treatment and itemized-deduction total." +us,scenario_099,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented a $34,000 standard deduction instead of using $20,476.57 of itemized deductions and then overstated tax on its $130,894 taxable-income figure. It also applied a renter credit despite the household's income and property-tax facts and failed to subtract the applicable exemption credits." us,scenario_099,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_100,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied CHIP's age and upper-income criteria without first testing Medicaid eligibility. Child1 qualifies for Montana Medicaid under the OLDER_CHILD category, and that Medicaid eligibility makes the child categorically ineligible for CHIP regardless of being below the CHIP income ceiling." -us,scenario_100,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model treated Montana's 261% FPL CHIP ceiling as a sufficient eligibility test. It omitted the prerequisite that a CHIP child not qualify for Medicaid; child1 qualifies under the OLDER_CHILD Medicaid category and therefore is not CHIP-eligible. -us,scenario_100,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model concluded that age and income below the CHIP maximum establish eligibility, skipping the Medicaid-exclusion step. Child1 is Medicaid-eligible under Montana's OLDER_CHILD category, which categorically precludes CHIP eligibility." -us,scenario_100,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated qualification for “Medicaid/CHIP coverage” with eligibility for the distinct CHIP output. Child1's very low income establishes Medicaid eligibility under the OLDER_CHILD category, and Medicaid eligibility makes the child ineligible for CHIP." -us,scenario_100,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stated the correct condition—CHIP applies if the child is not on Medicaid—but failed to apply it. At this income, child1 qualifies for Montana Medicaid under the OLDER_CHILD category, so the condition for CHIP is not met." -us,scenario_100,child1_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly placed child1 below Montana's Medicaid threshold but then mislabeled Medicaid-level coverage as CHIP eligibility. PolicyEngine treats child1 as Medicaid-eligible under the OLDER_CHILD category, and eligibility for Medicaid excludes the child from the separate CHIP program." -us,scenario_100,child1_head_start_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model misapplied the Head Start age pathway by extending preschool Head Start eligibility to a 6-year-old based on a broad “up to compulsory school age” rule. PolicyEngine's preschool Head Start output does not qualify Child 1 at age 6, so its income-based eligibility conclusion is irrelevant for this variable." -us,scenario_100,child1_head_start_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model treated age 6 as still inside the preschool Head Start range by assuming the child could be age 5 and not yet in kindergarten, even though the facts state age 6 and give no kindergarten exception. PolicyEngine's preschool Head Start eligibility pathway excludes Child 1 at age 6, so the low-income calculation does not produce eligibility." -us,scenario_100,child1_head_start_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model collapsed age 6 into a generic preschool or early-school-age category and then assumed low income was sufficient. PolicyEngine's requested Head Start variable is the preschool-age child pathway, and Child 1's age 6 status fails that eligibility step." -us,scenario_100,child1_head_start_eligible,grok-4.3,llm_error,age_disability,False,"The model asserted that age 6 meets Head Start criteria without applying the benchmark's preschool-age restriction. Under PolicyEngine's Head Start child output, Child 1 is not eligible at age 6, regardless of the household's low wages." -us,scenario_100,child1_head_start_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model incorrectly included a 6-year-old in the preschool Head Start age range by relying on a broad “up to compulsory school age” interpretation. PolicyEngine's preschool Head Start output does not qualify Child 1 at age 6, so the model's poverty-threshold reasoning cannot make the output yes." -us,scenario_100,child1_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model treated existing employer-sponsored insurance as a categorical Medicaid disqualifier for the child. PolicyEngine instead applies Montana's older child MAGI Medicaid pathway for a 6-year-old, and with MAGI at 0.07 of FPL against a 143% FPL limit, Child 1 is eligible despite having employer-sponsored insurance." -us,scenario_100,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to trigger the older child Medicaid category for a 6-year-old dependent in Montana. That pathway uses the child's MAGI-based household income test, and the 0.07 FPL income level is below Montana's 143% FPL older-child limit." -us,scenario_100,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable value or explanation for child1_medicaid_eligible. The required output was a numeric eligibility value, and the correct derivation returns 1 because Child 1 is a 6-year-old older-child Medicaid applicant with MAGI at 0.07 FPL, below Montana's 143% FPL limit." -us,scenario_100,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used the wrong WIC child age category by treating a 6-year-old as within the child WIC range. WIC child eligibility is limited to children under age 5, and this household also fails the WIC income test, so Child 1 is not eligible." -us,scenario_100,child1_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable answer or explanation for child1_wic_eligible. The required output was 0 because Child 1 is age 6, outside the WIC child category, and the household fails the WIC income test." -us,scenario_100,child2_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below Montana’s CHIP ceiling as sufficient for CHIP eligibility. It omitted the prior Medicaid screen: Child 2 qualifies for Medicaid under the YOUNG_CHILD category, which makes the child ineligible for CHIP." -us,scenario_100,child2_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model applied the CHIP income ceiling and uninsured requirement but failed to test Medicaid eligibility first. Child 2’s YOUNG_CHILD Medicaid eligibility precludes CHIP even though the child has no listed employer-sponsored insurance. -us,scenario_100,child2_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model concluded that very low income establishes CHIP eligibility, reversing the program coordination rule at this income level. Child 2 instead qualifies for Medicaid under the YOUNG_CHILD category, and that eligibility bars CHIP." -us,scenario_100,child2_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model checked age, Montana residence, income, and insurance status but omitted the required exclusion for children eligible for Medicaid. Child 2 qualifies for Medicaid under the YOUNG_CHILD category, so meeting the listed CHIP conditions does not produce CHIP eligibility." -us,scenario_100,child2_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stated the correct condition—CHIP applies only if the child is not on Medicaid—but never evaluated that condition and submitted the opposite result. Child 2 is Medicaid-eligible under the YOUNG_CHILD category, so CHIP eligibility is 0." -us,scenario_100,child2_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated being under 19, below the CHIP income threshold, and without employer-sponsored insurance as sufficient. It omitted Child 2’s Medicaid eligibility under the YOUNG_CHILD category, which disqualifies the child from CHIP." -us,scenario_100,child2_early_head_start_eligible,grok-4.3,llm_error,age_disability,False,"The model treated a 5-year-old as satisfying Early Head Start's age criterion. It relied on low income after passing the wrong age screen, but Early Head Start is limited to younger children, so Child 2's age alone rules out eligibility." -us,scenario_100,child2_head_start_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model imposed a kindergarten-age cutoff that excluded all age-5 children from preschool Head Start. It missed that this benchmark's Head Start output includes the age-5 child and that TANF receipt categorically qualifies the household, with income also below the applicable three-person threshold." -us,scenario_100,child2_head_start_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly listed Head Start enrollment or program flag as a negative eligibility determination. It failed to compute eligibility from the listed facts: TANF receipt creates categorical eligibility, and the household's adjusted gross income is below the Head Start limit for a three-person household." -us,scenario_100,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply the Montana young child Medicaid MAGI pathway for a 5-year-old dependent. It treated Medicaid eligibility as having no triggered basis from the listed facts, ignoring that child2's MAGI level is 0.07 times FPL and satisfies the young-child income test." -us,scenario_100,child2_wic_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated age 5 as inside PolicyEngine's WIC child category and then let the low income test decide eligibility. PolicyEngine requires the child categorical condition to be satisfied first, and child 2's listed age of 5 fails that condition." -us,scenario_100,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model misread WIC's age category as covering children aged 0 through 5 and added irrelevant asset reasoning. Child 2 is not categorically eligible for WIC in PolicyEngine at age 5, so income and assets do not produce eligibility." -us,scenario_100,child2_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model identified that age 5 reaches the WIC aging-out point, then overrode that rule by treating the child as still eligible because income was low. PolicyEngine applies the categorical age test, and child 2 fails WIC eligibility before the income test can matter." -us,scenario_100,child2_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated a 5-year-old as still categorically eligible through a WIC certification-period theory that is not provided in the benchmark facts. With constant annual facts and no within-year status changes, PolicyEngine evaluates child 2 as age 5 and not in the qualifying WIC child category." -us,scenario_100,child2_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model directly contradicted the age cutoff by calling age 5 under age 5. PolicyEngine's WIC child category is not satisfied for child 2 at age 5, so the household's low income does not establish eligibility." -us,scenario_100,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used a shortcut that age 5 is within the WIC child age range and relied on very low income. PolicyEngine's computation requires categorical WIC status, and child 2's age 5 fails that status." -us,scenario_100,child2_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable value or explanation for child2_wic_eligible. This is a missing-output failure rather than a substantive tax-benefit rule calculation. -us,scenario_100,child2_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model said age 5 is within an under-5 age limit, which reverses the categorical WIC age test. PolicyEngine treats child 2 at age 5 as not satisfying WIC categorical eligibility, despite income below the WIC threshold." -us,scenario_100,child2_wic_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated 'up to age 5' as including the listed age 5 for WIC eligibility. PolicyEngine evaluates child 2 as not meeting the WIC categorical eligibility condition, so the low wages do not make the child eligible." -us,scenario_100,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s reasoning derived approximately $2,878 from EITC plus refundable CTC, but it submitted $6,425. This violates the required contract that the numeric value match the explanation." -us,scenario_100,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model correctly derived about $2,366 of EITC but replaced the earnings-limited refundable CTC of $512.21 with an unsupported $834 approximation. The ACTC calculation is 15% of $5,915 minus $2,500, not a larger per-child estimate." -us,scenario_100,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model correctly computed 40% of $5,915 as roughly $2,366, then improperly reduced EITC to $1,906 by invoking the lower AGI. An AGI below earned income does not replace the earned-income phase-in calculation or reduce this credit." -us,scenario_100,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly identified approximately $2,366 of EITC and $512 of refundable CTC, then applied an unspecified reduction and submitted $2,330. No such reduction applies, and the submitted total is even lower than the EITC component alone." -us,scenario_100,federal_refundable_credits,claude-opus-5,llm_error,payroll_tax_base,False,"The model added the separately listed FLSA overtime premium to earned income even though the prompt states that $5,915 is the annual gross-wage total including overtime. This double counted compensation and inflated EITC from $2,365.89 to about $2,905." -us,scenario_100,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly derived about $2,366 of EITC and $512 of refundable CTC, then discarded that computation and substituted an IRS-table maximum-style amount. At $5,915 the EITC remains in the 40% phase-in range, so the household does not receive the maximum EITC." -us,scenario_100,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted retirement contributions from gross wages to reduce EITC earned income to $928 and then denied refundable CTC. The credit calculations use $5,915 of earned wages, producing $2,365.89 of EITC and $512.21 of refundable CTC." -us,scenario_100,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $3,000 refundable-CTC earnings threshold instead of the applicable $2,500 threshold. That excluded 15% of an additional $500 of earnings and understated the refundable CTC by about $75." -us,scenario_100,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The answer treats low income as yielding the full EITC and refundable CTC maxima. At $5,915, EITC is still phasing in at 40%, and refundable CTC is limited to 15% of earnings above $2,500, yielding total credits of $2,878.10 rather than $6,848." -us,scenario_100,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated the stated employer-sponsored insurance premiums as pre-tax deductions that eliminate all earned income. The prompt supplies $5,915 as annual gross wages and does not say those premiums reduce the credit base, so both EITC and refundable CTC remain positive." -us,scenario_100,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly netted health-insurance premiums against the $5,915 annual gross wages and set earned income to zero. Those expenses do not erase the earned-income base used for EITC and refundable CTC." -us,scenario_100,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $3,000 refundable-CTC earnings threshold instead of $2,500. The correct threshold produces $512.21 of refundable CTC rather than $437.25." -us,scenario_100,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model awarded the per-child refundable CTC cap without first applying the earnings limitation and calculated EITC from reduced AGI. Refundable CTC is limited to 15% of earnings above $2,500, while EITC phases in from the full $5,915 of earned wages." -us,scenario_100,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated very low earnings as disqualifying the household from refundable credits. A head-of-household filer with two qualifying children and $5,915 of earnings is in the EITC phase-in range and also has earnings above the refundable-CTC threshold." -us,scenario_100,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable credit was triggered. Two qualifying children and $5,915 of wages generate both a phase-in EITC and an earnings-limited refundable CTC." -us,scenario_100,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model asserted a zero EITC phase-in despite $5,915 of earned income and two qualifying children. Applying the 40% phase-in produces $2,365.89 of EITC, with another $512.21 from refundable CTC." -us,scenario_100,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted the traditional 401(k) deferral from gross wages and used $2,056 as EITC earned income, then denied refundable CTC under a $3,000 threshold. The calculation uses $5,915 of earned wages and the $2,500 refundable-CTC threshold." -us,scenario_100,federal_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used post-deferral W-2 Box 1 wages of $2,056 as the credit base. The trace uses the full $5,915 of earned wages, which generates both $2,365.89 of EITC and $512.21 of refundable CTC." -us,scenario_100,federal_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model reduced EITC earned income to $2,056 for the traditional 401(k) deferral and consequently placed earnings below both candidate ACTC thresholds. The applicable earned-income amount is $5,915, and the refundable CTC threshold is $2,500." -us,scenario_100,federal_refundable_credits,minimax-m3,llm_error,other,False,"The explanation states that the household receives substantial positive EITC and refundable CTC, but the model submitted zero. The numeric output contradicts its own stated eligibility and estimated positive amounts." -us,scenario_100,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model treated the refundable CTC per-child cap as an automatic award and later replaced the correctly derived EITC phase-in amount with $3,730. The refundable CTC is instead constrained to 15% of earnings above $2,500, and EITC remains $2,365.89 at the 40% phase-in rate." -us,scenario_100,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly listed free-meal benefit as proof of ineligibility instead of computing program eligibility and assumed take-up. It missed automatic categorical qualification through $8,625.89 of annual SNAP receipt and also failed to apply the 130% poverty-guideline test, which the household passes at 44%." -us,scenario_100,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model used the wrong Medicaid pathway and arithmetic: it treated the head as a disability/expansion adult subject to a 138% FPL test, then misread 138% FPL for a household of three as about $3,627 annually. It also imposed an asset limit, even though the relevant PolicyEngine pathway is Montana's MAGI parent/caretaker category, where the head's 0.07 x FPL income is below the 100% FPL parent threshold." -us,scenario_100,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to recognize the parent/caretaker eligibility category created by the two dependent children in the household. Under Montana's Medicaid parent/caretaker rules, the head's MAGI of 0.07 x FPL is below the 100% FPL threshold, so the absence of a separate disability-specific trigger does not block eligibility." -us,scenario_100,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable value for head_medicaid_eligible. The required output is eligible because the head is a Montana parent/caretaker with MAGI at 0.07 x FPL, below the 100% FPL parent/caretaker threshold." -us,scenario_100,head_medicare_eligible,claude-fable-5,llm_error,health_coverage,False,"The model incorrectly treated disability status by itself as a Medicare-qualifying condition for an under-65 adult. It skipped the Medicare entitlement requirement and converted the listed `is disabled` fact directly into eligibility, yielding 1 instead of the age/qualifying-condition result of 0." -us,scenario_100,head_medicare_eligible,gpt-5.5,llm_error,health_coverage,False,"The model invented a simplified PolicyEngine Medicare rule under which any disabled person qualifies regardless of age. PolicyEngine does not use the raw disability flag alone as Medicare eligibility for this 46-year-old head, so the correct computation leaves head_medicare_eligible at 0." -us,scenario_100,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model converted a child's assumed WIC eligibility into the head's eligibility, treating WIC as adjunctively conferred to the household. WIC categorical eligibility is individual; a 46-year-old head with no pregnancy, postpartum, or breastfeeding status is not WIC-eligible merely because the household has children and low income." -us,scenario_100,head_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model misread the parent or caretaker of a young child as a WIC categorical group and also treated age 5 as a qualifying under-5 child category. WIC eligibility for the head requires the head to be pregnant, postpartum, or breastfeeding, and those facts are false under the prompt." -us,scenario_100,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented a caregiver-related WIC category for a low-income parent with children. The head has no listed pregnancy, postpartum, breastfeeding, infant, or under-5 child status applying to the head, so income alone cannot make the head WIC-eligible." -us,scenario_100,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable answer or explanation for head_wic_eligible. The correct output is 0 because the head lacks an individual WIC categorical eligibility status, so the missing response failed the output contract rather than applying a substantive rule." +us,scenario_100,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied CHIP's age and upper-income criteria without first excluding children eligible for Medicaid. Child 1 qualifies for Medicaid under Montana's OLDER_CHILD category, which makes the child categorically ineligible for CHIP regardless of employer-sponsored insurance." +us,scenario_100,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model treated income below Montana's CHIP ceiling as sufficient for CHIP eligibility. It omitted the Medicaid-exclusion step: Child 1 qualifies for Medicaid under the OLDER_CHILD category and therefore cannot qualify for CHIP. +us,scenario_100,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model compared household income only with the CHIP upper limit and concluded that lower income establishes eligibility. At this income, the age-6 child instead qualifies for Medicaid under the OLDER_CHILD category, and that Medicaid eligibility precludes CHIP." +us,scenario_100,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model collapsed Medicaid and CHIP into a single combined eligibility result, stating that the child qualifies for “Medicaid/CHIP.” The requested CHIP indicator is mutually exclusive with Medicaid eligibility here: Child 1 qualifies for Medicaid under the OLDER_CHILD category and is consequently ineligible for CHIP." +us,scenario_100,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stated the controlling condition—CHIP applies only if the child is not on Medicaid—but failed to evaluate it. Child 1 is Medicaid-eligible under Montana's OLDER_CHILD category, so the model's own stated CHIP condition is not satisfied." +us,scenario_100,child1_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly placed the child below the Medicaid income limit but then mislabeled Medicaid coverage as “Medicaid-level CHIP coverage” and returned CHIP eligibility. PolicyEngine classifies Child 1 as Medicaid-eligible under the OLDER_CHILD category, and Medicaid eligibility categorically excludes the child from CHIP." +us,scenario_100,child1_head_start_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model incorrectly treated age 6 as eligible merely because Head Start can serve children until compulsory school age. Under the benchmark's preschool-age test, Child 1 fails the age requirement, so the family's low income cannot produce eligibility." +us,scenario_100,child1_head_start_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model invented an unlisted circumstance in which the six-year-old had only recently turned five and was not yet in kindergarten. The prompt fixes the child's age at 6 for the full year and makes unlisted statuses false, so Child 1 does not satisfy the Head Start preschool-age test." +us,scenario_100,child1_head_start_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model replaced the program's specific preschool-age requirement with the broader label ""preschool/early-school age."" A six-year-old fails the benchmark's Head Start age test regardless of low income." +us,scenario_100,child1_head_start_eligible,grok-4.3,llm_error,age_disability,False,"The model asserted that age 6 met Head Start criteria without applying the preschool-age cutoff. Child 1 is outside the qualifying age range, so low household income does not establish eligibility." +us,scenario_100,child1_head_start_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model improperly extended the ordinary ages-3-to-5 rule through compulsory-school age and therefore counted the six-year-old as age-eligible. The benchmark's Head Start preschool-age test excludes Child 1, independently of the income result." +us,scenario_100,child1_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model incorrectly treated employer-sponsored insurance as a categorical Medicaid disqualifier. Montana's older-child Medicaid pathway applies a 143% FPL MAGI limit, and this child's 0.07 × FPL income satisfies it regardless of the listed employer-sponsored coverage." +us,scenario_100,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model failed to apply Montana's older-child Medicaid category to the 6-year-old dependent. That pathway compares the child's 0.07 × FPL MAGI with the 143% FPL limit and yields eligibility. +us,scenario_100,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for the requested output, violating the required output contract." +us,scenario_100,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 6 as within WIC's child eligibility range. WIC covers children under age 5, so Child 1 fails the categorical age test; the model also incorrectly described the traced household income as within the WIC limit." +us,scenario_100,child1_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output for child1_wic_eligible, violating the required submission contract." +us,scenario_100,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The model incorrectly placed a six-year-old within WIC's child age range and incorrectly treated the traced income as within the program limit. Child 1 is not under age 5 and the traced income exceeds the applicable 185%-of-FPL threshold. +us,scenario_100,child2_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model treated income below Montana's CHIP ceiling as sufficient for CHIP eligibility. It failed to apply the Medicaid-exclusion step: child2 qualifies for Medicaid under the YOUNG_CHILD category and therefore is not eligible for CHIP. +us,scenario_100,child2_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model checked the CHIP income ceiling and uninsured status but omitted the requirement that a CHIP applicant not qualify for Medicaid. Child2's very low household income establishes Medicaid eligibility under the YOUNG_CHILD category, which blocks CHIP eligibility." +us,scenario_100,child2_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly converted income below the CHIP limit directly into CHIP eligibility. It skipped child2's Medicaid determination: the age-5 child is Medicaid-eligible under the YOUNG_CHILD category, so CHIP is unavailable." +us,scenario_100,child2_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model treated age, income, Montana residency, and lack of employer-sponsored insurance as a complete CHIP eligibility test. It omitted the controlling Medicaid screen; child2 qualifies for Medicaid under the YOUNG_CHILD category, and that eligibility precludes CHIP." +us,scenario_100,child2_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stated the correct condition—CHIP applies only if the child is not on Medicaid—but returned yes without performing that condition's eligibility test. PolicyEngine determines that child2 is Medicaid-eligible under the YOUNG_CHILD category, so the correct application of the model's own rule yields no CHIP eligibility." +us,scenario_100,child2_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model relied on income below Montana's CHIP threshold and the absence of employer-sponsored insurance while omitting Medicaid eligibility. Child2 qualifies for Medicaid under the YOUNG_CHILD category, which makes the child ineligible for CHIP regardless of satisfying the CHIP ceiling or being uninsured." +us,scenario_100,child2_early_head_start_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly stated that age 5 meets the Early Head Start age criteria. Early Head Start is limited to children under age 3, so Child 2 fails the age test regardless of the household's low income." +us,scenario_100,child2_head_start_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model incorrectly treated age five as an automatic Head Start cutoff based on typical kindergarten entry. Child 2 remains within the applicable preschool Head Start age rule and qualifies through household TANF receipt, with income also below the three-person threshold." +us,scenario_100,child2_head_start_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model asserted a negative result without applying either eligibility pathway. Household TANF receipt makes Child 2 categorically eligible, and adjusted gross income after the pre-tax 401(k) contribution and traditional IRA deduction also passes the income test." +us,scenario_100,child2_head_start_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model misapplied the Head Start age rule by declaring every five-year-old outside the preschool eligibility range. Child 2 satisfies the applicable age condition and qualifies categorically through TANF receipt, as well as under the income threshold." +us,scenario_100,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply Montana's YOUNG_CHILD Medicaid category to the 5-year-old dependent. At a MAGI income level of 0.07 times FPL, child2 satisfies the category's income test and is Medicaid eligible." +us,scenario_100,child2_wic_eligible,claude-fable-5,llm_error,categorical_eligibility,False,The model incorrectly treated age 5 as within PolicyEngine's under-five WIC child category. It also presumed nutritional risk despite the instruction that unlisted statuses are false; low income does not cure the failed categorical test. +us,scenario_100,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model interpreted the WIC child range as inclusive of age 5, but child eligibility ends at the fifth birthday. Its asset-limit analysis is irrelevant to this failed categorical test and also invents a WIC asset screen." +us,scenario_100,child2_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model explicitly recognized that a child aged 5 has reached the fifth birthday and aged out, then reversed that conclusion because income was low. Income eligibility cannot override the under-five categorical requirement." +us,scenario_100,child2_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model incorrectly converted eligibility through the fifth birthday into eligibility after the child is already age 5. Certification-period continuation was not supplied and cannot be inferred under the prompt's constant-status and unlisted-facts rules. +us,scenario_100,child2_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model called a five-year-old “under age 5” and therefore applied the WIC child category incorrectly. The income test is immaterial once the age-based categorical requirement fails. +us,scenario_100,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 5 as included in the WIC child age range. PolicyEngine's categorical test requires the child to be under five, so the very low income does not produce eligibility." +us,scenario_100,child2_wic_eligible,inkling,llm_error,categorical_eligibility,False,The model relied on age 5 and income below 185% of poverty without enforcing the under-five categorical requirement. A child already aged 5 is outside the WIC child category. +us,scenario_100,child2_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child2_wic_eligible, violating the required output contract." +us,scenario_100,child2_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model described a child aged 5 as “under 5,” which is the exact categorical error. Falling below the income threshold does not establish WIC eligibility for a child who has aged out." +us,scenario_100,child2_wic_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model interpreted “up to age 5” as including children who are already 5. WIC child categorical eligibility ends at the fifth birthday, regardless of the household's low income." +us,scenario_100,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model included age 5 in the WIC child category and then relied on the income limit. Because the child is not under five, the categorical test fails before income can establish eligibility." +us,scenario_100,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived approximately $2,878 from $2,366 of EITC and $512 of refundable CTC, then submitted $6,425 instead. Its numeric output contradicts its completed calculation." +us,scenario_100,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model correctly calculated the refundable CTC earnings limit as $512.25, then replaced it with an unsupported $834 approximation. Adding $2,365.89 of EITC to $512.21 of refundable CTC yields $2,878.10." +us,scenario_100,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model arbitrarily reduced the EITC from its own correct 40% phase-in calculation of about $2,366 to $1,906. The lower AGI does not reduce the credit in this phase-in range; the EITC is $2,365.89 and refundable CTC is $512.21." +us,scenario_100,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly derived about $2,366 of EITC and $512 of refundable CTC but then applied an unspecified reduction and submitted $2,330. No such reduction applies, so the two components sum to $2,878.10." +us,scenario_100,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the separately listed FLSA overtime premium to earned income even though gross wages already include all overtime pay. Using the stated annual wages once produces $2,365.89 of EITC, not $2,905." +us,scenario_100,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model abandoned its correct phase-in calculation of roughly $2,366 of EITC plus $512 of refundable CTC and substituted an EITC-table maximum-style total of $6,295. At $5,915 of earnings, the household remains in the 40% EITC phase-in and cannot receive the maximum credit." +us,scenario_100,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced EITC earned income by retirement and other deductions to $928. The relevant earned income is the $5,915 of wages, producing $2,365.89 of EITC and also exceeding the $2,500 refundable-CTC threshold by $3,415." +us,scenario_100,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $3,000 refundable-CTC earned-income threshold instead of the applicable $2,500 threshold. Applying 15% to $5,915 minus $2,500 produces $512.21 rather than $437.25." +us,scenario_100,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The answer treats low income as generating full refundable credits. The EITC is still in its 40% phase-in and the refundable CTC is restricted to 15% of earnings above $2,500, yielding $2,365.89 and $512.21 rather than $6,848." +us,scenario_100,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly netted the listed employer-sponsored insurance premiums against wages and set EITC earned income to zero. The prompt states $5,915 as annual gross wages, which supports both $2,365.89 of EITC and $512.21 of refundable CTC." +us,scenario_100,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated health-insurance premiums as eliminating the household's earned income. The $5,915 gross-wage input remains the earnings base for the credit calculations, generating positive EITC and refundable CTC." +us,scenario_100,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $3,000 refundable-CTC threshold rather than $2,500. The correct earnings-limited refundable CTC is $512.21, which combines with $2,365.89 of EITC." +us,scenario_100,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model awarded the $1,700-per-child refundable CTC cap without applying the 15% earnings limitation, and it calculated EITC from reduced AGI rather than the $5,915 earnings base. The earnings limitation restricts refundable CTC to $512.21, while the EITC phase-in produces $2,365.89." +us,scenario_100,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated very low earnings as disqualifying the household from refundable credits. Two qualifying children and $5,915 of earnings generate a phase-in EITC of $2,365.89 and refundable CTC of $512.21." +us,scenario_100,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable-credit provision was triggered. The household satisfies the two-child EITC pathway and has earnings above the refundable-CTC threshold, producing total credits of $2,878.10." +us,scenario_100,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly assigned a zero EITC phase-in amount despite $5,915 of earned income and two qualifying children. The 40% phase-in produces $2,365.89, and earnings above $2,500 also produce $512.21 of refundable CTC." +us,scenario_100,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the traditional 401(k) deferral from the EITC and refundable-CTC earned-income base. Using the full $5,915 of wages produces $2,365.89 of EITC and places earnings above the $2,500 refundable-CTC threshold." +us,scenario_100,federal_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly used wages after the traditional 401(k) deferral as earned income for both credits. The applicable $5,915 earnings base yields $2,365.89 of EITC and $512.21 of refundable CTC." +us,scenario_100,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced earned income by both the traditional 401(k) and IRA contributions. EITC and the refundable-CTC earnings test use the $5,915 wage amount here, not the resulting $1,876 AGI estimate." +us,scenario_100,federal_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly used W-2 Box 1 wages after the 401(k) deferral as the earned-income base. The $5,915 wage input produces $2,365.89 of EITC and exceeds the applicable $2,500 refundable-CTC threshold." +us,scenario_100,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model's reasoning states that substantial EITC and refundable CTC apply, but it submitted zero. Bank assets and the standard deduction do not cancel these federal refundable credits; the applicable components total $2,878.10." +us,scenario_100,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model treated the per-child refundable CTC ceiling as an automatic award and later substituted an unsupported $3,730 EITC. The refundable CTC is instead earnings-limited to $512.21, and the 40% EITC phase-in produces $2,365.89." +us,scenario_100,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly capped refundable credits by income-tax liability and concluded that low earnings produce zero. EITC and refundable CTC are payable beyond tax liability, with the stated earnings generating $2,365.89 and $512.21 respectively." +us,scenario_100,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model asserted that positive support was not indicated instead of deriving program receipt and income from the supplied household facts under the assumed-take-up instruction. It missed both automatic categorical eligibility through $8,625.89 of annual SNAP and the independent income pathway, under which countable income equals 44% of the federal poverty guideline and is below the 130% free-meal threshold." +us,scenario_100,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model evaluated the head only under an expansion/disability pathway and missed the parent/caretaker category, whose Montana income limit is 100% FPL and which the head satisfies at 0.07 × FPL. It also used an erroneous $3,627 annual 138%-FPL threshold and incorrectly imposed an asset limit on MAGI-based Medicaid eligibility." +us,scenario_100,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the parent/caretaker pathway established by the head living with two dependent children. Under that category, the head’s MAGI of 0.07 × FPL is below Montana’s 100%-FPL limit, and neither separate disability-based expansion nor lack of other coverage is required." +us,scenario_100,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output for head_medicaid_eligible. The required result is 1 because the head qualifies under Montana’s parent/caretaker category at 0.07 × FPL, below the 100%-FPL threshold." +us,scenario_100,head_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated disability as independently sufficient for Medicare eligibility at any age. It failed to require a Medicare-qualifying disability entitlement or another under-65 pathway, none of which is listed for the 46-year-old head." +us,scenario_100,head_medicare_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model invented a simplified PolicyEngine rule under which the disability flag automatically confers Medicare eligibility. PolicyEngine does not convert an unqualified disability input into Medicare eligibility without the required under-65 qualifying condition or entitlement. +us,scenario_100,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated a child's potential WIC eligibility as adjunctively conferring WIC eligibility on the household head. WIC does not extend individual eligibility to a parent or caretaker; the head is not stated to be pregnant, breastfeeding, or postpartum." +us,scenario_100,head_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model replaced WIC's individual categorical requirement with a parent-or-caretaker pathway that does not exist. It also treated age 5 as under age 5, but neither a five-year-old child nor low income makes the adult head categorically eligible." +us,scenario_100,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model explicitly acknowledged that no pregnant, postpartum, breastfeeding, or other qualifying adult category was stated, then improperly substituted low-income parenthood. A parent or caregiver is not personally WIC-eligible without an eligible adult category." +us,scenario_100,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for head_wic_eligible, so it failed the required output contract." us,scenario_100,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_100,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model incorrectly subtracted health insurance premiums from FICA taxable wages before computing employee Social Security and Medicare tax. PolicyEngine applies the 6.2% Social Security and 1.45% Medicare rates to the head's FICA wage base, yielding $366.71 plus $85.76 rather than taxing $4,787." -us,scenario_100,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated employer-sponsored and other health insurance premiums as pre-tax deductions that fully eliminate the FICA wage base. Payroll tax in this benchmark is assessed on the head's wages, so the premiums do not reduce taxable wages to zero and the employee FICA total is $452.48." -us,scenario_100,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model zeroed out employee FICA wages by subtracting pre-tax employer-sponsored insurance premiums from gross wages. PolicyEngine taxes the head's wage base for Social Security and Medicare, so the correct result includes $366.71 of Social Security tax and $85.76 of Medicare tax." -us,scenario_100,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model identified the correct employee-side FICA pathway but miscomputed the combined 7.65% tax on the wage base. Applying 6.2% Social Security and 1.45% Medicare to the PolicyEngine wage base gives $452.48, not $453.55." -us,scenario_100,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's stated FICA pathway is right, but its arithmetic is wrong and its final answer discards its own near-correct subtotal. Social Security plus Medicare on the PolicyEngine wage base is $452.48, while $477 is not produced by the 6.2% and 1.45% rates it cited." -us,scenario_100,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied only the 185% upper limit and treated income below that limit as sufficient for reduced-price meals. It failed to classify the household into the superior FREE tier based on its 44% FPG income and categorical eligibility, which makes reduced-price support inapplicable." -us,scenario_100,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly stated that $5,915 was above 130% FPG even though the engine's household-size calculation gives a 0.44 FPG ratio. It therefore assigned the 130%-to-185% reduced-price band instead of the FREE tier, while also failing to apply categorical free-meal eligibility." -us,scenario_100,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used wages alone as gross income and substituted a $93.67 medical deduction for the applicable deduction stack, omitting the $453.64 excess-shelter deduction and the other income included in the $998.31 monthly gross amount. Those errors produced $89 of net income instead of $237.09 and understated the required $71.10 contribution." -us,scenario_100,snap,claude-haiku-4.5,llm_error,other,False,"The model established eligibility but never calculated the allotment from net income and the three-person maximum; its unexplained $1,908 is not the result of any SNAP benefit formula stated in its reasoning. The correct computation subtracts a $71.10 monthly contribution from the applicable monthly maximums and sums the resulting allotments to $8,625.89." -us,scenario_100,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model counted wages alone, constructed an inapplicable $141 monthly medical deduction, and explicitly treated shelter deductions as absent. The trace instead applies $761.21 of total deductions, including $453.64 of excess shelter costs, to $998.31 of gross income, leaving $237.09 net." -us,scenario_100,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model reduced annual wages by only an earned-income and estimated standard deduction, then approximated the result without applying the traced $453.64 excess-shelter deduction or the full $998.31 monthly gross-income base. SNAP net income is $237.09, so the household contribution is $71.10 rather than the model's estimated $58." -us,scenario_100,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,The model used wages alone and an unspecified 'very low' net-income estimate instead of calculating $998.31 gross income minus $761.21 in allowable deductions. It therefore failed to derive the $237.09 net income and $71.10 contribution used in each monthly allotment. -us,scenario_100,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model repeatedly changed estimated maximum allotments and computed net income from wages, a guessed standard deduction, and medical expenses while assigning no shelter deduction. The traced calculation uses $998.31 gross income, a $209 standard deduction, a $98.58 earned-income deduction, and a $453.64 excess-shelter deduction, producing $237.09 net income." -us,scenario_100,snap,claude-sonnet-5,llm_error,period_annualization,False,"The model treated deductible expenses as eliminating net income and then annualized inconsistently: $766 per month for 12 months equals $9,192, not $13,700. Net income is $237.09 rather than zero, and the changing monthly maximums must be reduced by the $71.10 contribution and summed across 2026." -us,scenario_100,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model included TANF in gross income but allowed only $46.67 of medical expense and omitted the $453.64 excess-shelter deduction, leaving $602.21 of net income. The applicable deductions total $761.21 and reduce $998.31 of gross income to $237.09." -us,scenario_100,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used wages alone and a $140 medical deduction to derive $54 of net income, omitting the actual gross-income components and the traced $453.64 excess-shelter deduction. The correct net income is $237.09, which creates a $71.10 monthly contribution." -us,scenario_100,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer equals 12 times the $784 maximum allotment and therefore gives the household the full maximum without subtracting its expected contribution. SNAP requires subtracting $71.10, equal to 30% of $237.09 net income, while also applying the later-month maximum update." -us,scenario_100,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model described a near-maximum benefit but used an annual figure equivalent to only about $510.67 per month, which is incompatible with the applicable three-person maximums and low net income. Subtracting the $71.10 contribution from each applicable monthly maximum yields $8,625.89." -us,scenario_100,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Although the model included TANF, it did not apply the traced $761.21 deduction total, especially the $453.64 excess-shelter deduction. The resulting allotment must be based on $237.09 net income and a $71.10 contribution, not the unstated higher net income embedded in $7,257." -us,scenario_100,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,The model based gross income on $492.92 of monthly wages and did not account for the full $998.31 gross-income amount in the trace. Applying $761.21 of deductions to the full gross amount leaves $237.09 net income and prevents the larger benefit it estimated. -us,scenario_100,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $20,644 employer-sponsored insurance premium as a household-paid SNAP medical expense and also counted medical expenses reported for a child as the disabled head's deductible costs. Net income is $237.09 rather than zero, so the household receives less than the maximum allotment." -us,scenario_100,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model assigned a purported full maximum of $12,588, equivalent to $1,049 per month, which is not the three-person SNAP maximum used in 2026. The applicable maximums are $785 and $802.60, and each is reduced by the $71.10 household contribution." -us,scenario_100,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly made missing housing details a bar to SNAP and returned zero despite the household satisfying categorical eligibility through TANF non-cash status and every income and asset test. The trace supplies a $453.64 excess-shelter deduction and produces positive monthly allotments totaling $8,625.89." -us,scenario_100,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model treated reported health expenses as reducing countable net income to zero and awarded a full estimated maximum. The applicable deductions leave $237.09 of net income, requiring a $71.10 reduction from each month's maximum." -us,scenario_100,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model multiplied the $785 maximum by 12 without subtracting the expected household contribution. The benefit equals the applicable monthly maximum minus $71.10, with later months using $802.60, and the annual sum is $8,625.89." -us,scenario_100,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model concluded that medical deductions reduced net income to zero and therefore awarded the full $785 maximum each month. The traced deductions leave $237.09 of net income, so SNAP subtracts a $71.10 monthly contribution." -us,scenario_100,snap,gpt-5.6-terra,llm_error,other,False,"The model named earnings, SSI, standard deductions, and medical deductions but supplied no calculation supporting its $6,400 estimate and omitted the decisive excess-shelter deduction. The traced formula yields $237.09 net income, a $71.10 contribution, and $8,625.89 after summing monthly allotments." -us,scenario_100,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model returned zero without applying SNAP eligibility or benefit rules. The household passes the income and asset tests, satisfies categorical eligibility through TANF non-cash status, and receives positive monthly allotments totaling $8,625.89." -us,scenario_100,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model treated $668 as annual net income after using wages alone, a $198 standard deduction, and a medical deduction, while omitting the traced $453.64 monthly excess-shelter deduction and the full $998.31 monthly gross income. The applicable monthly net income is $237.09, producing a $71.10 contribution." -us,scenario_100,snap,grok-build-0.1,llm_error,period_annualization,False,The model reported a single $805 monthly maximum as the annual SNAP output and also incorrectly reduced net income to zero. The annual output requires subtracting the $71.10 contribution from each month's applicable maximum and summing all 12 months. -us,scenario_100,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no SNAP value or explanation, so it failed the required output contract before any substantive calculation could be evaluated." -us,scenario_100,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model treated premiums and medical expenses as reducing SNAP net income to zero and multiplied the $785 maximum by 12. The applicable deduction calculation leaves $237.09 net income, requiring a $71.10 contribution in every month." -us,scenario_100,snap,minimax-m3,llm_error,thresholds_rates,False,"The model first recognized a near-maximum benefit but then subtracted roughly $389 per month from the maximum despite stating net income of only about $393; SNAP subtracts 30% of net income, not nearly 100%. The traced net income is $237.09 and its 30% contribution is $71.10." -us,scenario_100,snap,qwen-3.7-max,llm_error,period_annualization,False,"The model double-counted the $710 FLSA overtime premium by adding it to wages even though gross wages already include overtime, and its annualization is arithmetically impossible because $766 times 12 is $9,192, not $12,444. The calculation must use the traced $998.31 monthly gross income, subtract $761.21 of deductions, and sum the monthly maximums net of the $71.10 contribution." -us,scenario_100,ssi,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the head’s generic disability flag as SSI categorical eligibility and proceeded directly to the earned-income calculation. The head’s SSI-specific aged/blind/disabled indicator is false, so no SSI benefit-rate or income-exclusion calculation applies." -us,scenario_100,ssi,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model equated `is disabled` with satisfaction of SSI’s categorical disability requirement. No household member has the SSI-specific aged, blind, or disabled status, so the individual SSI amounts are all zero." -us,scenario_100,ssi,gpt-5.5,llm_error,categorical_eligibility,False,"The model assumed the head was an SSI-eligible disabled adult and subtracted countable earnings from a federal benefit maximum. The head fails SSI’s categorical gate, so the federal rate, earned-income exclusions, and Montana supplement are never reached." -us,scenario_100,ssi,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model declared the disabled head SSI-eligible without applying the SSI-specific aged/blind/disabled indicator. That indicator is false for the head and both children, producing zero before any earned-income adjustment." -us,scenario_100,ssi,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model treated the head as categorically eligible and computed federal SSI as the individual maximum less countable earnings. The head does not meet SSI’s aged, blind, or disabled criterion under the engine inputs, so the correct computation stops at zero." -us,scenario_100,ssi,grok-build-0.1,llm_error,categorical_eligibility,False,"The model correctly described an earned-income exclusion calculation for an eligible recipient but wrongly applied it to the head. The head’s SSI-specific categorical eligibility flag is false, making the federal benefit rate and Montana supplement irrelevant." -us,scenario_100,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable SSI value or explanation. It therefore failed the required output contract rather than completing the categorical-eligibility calculation. -us,scenario_100,ssi,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model assumed the generic disability fact qualified the head for SSI and then performed earnings, exclusion, and IRWE adjustments. The head fails SSI’s categorical eligibility gate, and no listed IRWE can be inferred; SSI is zero without reaching those adjustments." -us,scenario_100,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted desired traditional 401(k) contributions from EITC earned income and then applied an obsolete 10% Montana rate. PolicyEngine uses $5,915 of earned income to produce a $2,365.89 federal EITC, then applies Montana's 20% rate for $473.18." -us,scenario_100,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Montana has no applicable refundable EITC. This filer qualifies for the refundable Montana EITC, equal to 20% of the $2,365.89 federal credit." -us,scenario_100,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model identified the two-child federal EITC but used a 10% Montana rate instead of 20% and then submitted an amount inconsistent with its own $237 calculation. The correct multiplication is $2,365.89 × 20% = $473.18." -us,scenario_100,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model acknowledged a positive federal EITC and an applicable refundable Montana EITC, then discarded that credit and submitted zero. The applicable 20% state credit on $2,365.89 is $473.18." -us,scenario_100,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly classified Montana's EITC as nonrefundable. In 2026 it is refundable at 20% of the federal EITC, yielding $473.18 regardless of zero pre-credit state liability." -us,scenario_100,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly derived a federal EITC of about $2,366 but applied a 10% Montana rate instead of the 2026 rate of 20%. Applying 20% to $2,365.89 yields $473.18." -us,scenario_100,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly treated Montana's EITC as nonrefundable and therefore eliminated it when state liability was zero. The refundable credit equals 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used both an erroneous $371 federal EITC and an obsolete 10% Montana rate. Two qualifying children and $5,915 of earnings generate $2,365.89 federally, and Montana refunds 20%, producing $473.18." -us,scenario_100,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly estimated the federal EITC near $2,366 but applied a 10% Montana percentage instead of 20%. The correct state credit is $2,365.89 × 20% = $473.18." -us,scenario_100,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted the refundable Montana EITC that applies to this low-income filer with two qualifying children. That credit is 20% of $2,365.89, yielding $473.18." -us,scenario_100,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model incorrectly asserted both that Montana lacks a refundable EITC and that the filer has zero earned income. The listed $5,915 of wages generates a $2,365.89 federal EITC, of which Montana refunds 20%." -us,scenario_100,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly set the federal EITC to zero despite $5,915 of earnings and two qualifying children. The federal credit is $2,365.89, making the 20% Montana EITC $473.18." -us,scenario_100,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model falsely stated that Montana has no individual income tax and consequently omitted its refundable EITC. Montana's applicable credit is 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to apply Montana's refundable EITC to a filer with positive earnings and two qualifying children. The 20% state share of the $2,365.89 federal EITC is $473.18." -us,scenario_100,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that the benchmark produces no refundable Montana credit. The household qualifies for a $473.18 Montana EITC, calculated as 20% of its $2,365.89 federal EITC." -us,scenario_100,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model overstated the federal EITC as $4,207 and applied an obsolete 10% Montana rate. At $5,915 of earnings the two-child phase-in produces $2,365.89 federally, and the 20% Montana credit is $473.18." -us,scenario_100,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model correctly identified a federal EITC near $2,366 but used a 10% Montana rate instead of 20%. The resulting refundable state credit is $473.18." -us,scenario_100,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model invented a $1,200 Montana young-child credit for the five-year-old and also used a 10% rather than 20% EITC rate. The only refundable state component here is the Montana EITC of $473.18." -us,scenario_100,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model incorrectly added a $1,200 young-child credit that does not apply in this computation and halved the Montana EITC by using a 10% rate. The sole refundable state credit is 20% of $2,365.89, or $473.18." -us,scenario_100,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model omitted the applicable refundable Montana EITC. Two qualifying children and $5,915 of earnings produce a $2,365.89 federal EITC and a $473.18 Montana credit." -us,scenario_100,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly stated that Montana offers no state EITC. Montana provides a refundable credit equal to 20% of this household's $2,365.89 federal EITC, yielding $473.18." -us,scenario_100,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model failed to apply the refundable Montana EITC for which this working family qualifies. The credit equals $2,365.89 × 20% = $473.18." -us,scenario_100,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model incorrectly asserted that Montana lacks a refundable state EITC and focused only on an inapplicable property-tax circuit breaker. The applicable Montana EITC is 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly derived the federal EITC near $2,366 but applied a 10% Montana rate instead of 20%. The correct refundable credit is $473.18." -us,scenario_100,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely stated that Montana has no state income tax and therefore omitted the state's refundable EITC. The applicable credit is 20% of $2,365.89, producing $473.18." -us,scenario_100,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model incorrectly stated that Montana has no EITC and treated zero tax liability as eliminating refundable credits. Montana's refundable EITC applies independently of pre-credit liability and equals $473.18. -us,scenario_100,tanf,claude-fable-5,llm_error,thresholds_rates,False,"The model first calculated positive eligibility and then contradicted that calculation by declaring the grant zero. It used speculative $588 and $633 monthly standards and an incorrect $200-plus-25% disregard calculation instead of Montana's 2026 formula yielding $6,064.96." -us,scenario_100,tanf,claude-haiku-4.5,llm_error,asset_resource,False,"The model used an inapplicable $2,000 asset limit and even described the three-person household as a household of four. The $2,800 bank balance and excluded vehicle do not disqualify this assistance unit, and its $5,915 income remains within Montana's applicable TANF limits." -us,scenario_100,tanf,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model acknowledged low income, qualifying children, assumed take-up, and assets within the limits, yet set the benefit to zero based on unspecified work and income concerns. Disability and the stated employment pattern do not erase financial eligibility, and Montana's benefit formula produces $6,064.96." -us,scenario_100,tanf,claude-opus-4.8,llm_error,thresholds_rates,False,"The model estimated roughly $595 per month from an unsupported maximum and did not correctly reduce or determine the grant under Montana's 2026 formula. The applicable calculation yields about $505.41 per month, totaling $6,064.96." -us,scenario_100,tanf,claude-opus-5,llm_error,categorical_eligibility,False,"The answer simply treated the household as ineligible despite two dependent children and income well below Montana's TANF thresholds. Applying the modeled eligibility and benefit rules yields $6,064.96 rather than zero." -us,scenario_100,tanf,claude-sonnet-4.6,llm_error,asset_resource,False,"The model applied an outdated or inapplicable $1,000 liquid-asset limit. Montana's modeled resource rules do not disqualify this household for its $2,800 bank balance, while the vehicle is excluded, so the benefit calculation continues to $6,064.96." -us,scenario_100,tanf,claude-sonnet-5,llm_error,thresholds_rates,False,"The model substituted an approximately $327 monthly maximum payment standard and generic 20% and $200 disregards. Montana's 2026 three-person standard and benefit calculation instead produce $6,064.96 annually." -us,scenario_100,tanf,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model imposed a $605 maximum and a $200-plus-50% earned-income disregard, then subtracted $146.46 monthly. Those are not the applicable Montana 2026 computation parameters; the correct state formula yields $6,064.96." -us,scenario_100,tanf,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $588 monthly standard and treated 75% of earnings above $200 as countable, producing $368 monthly. It also rounded $4,416 to an unrelated $4,419; Montana's applicable 2026 formula yields $6,064.96." -us,scenario_100,tanf,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model denied eligibility without identifying any failed Montana requirement. The household contains a qualifying parent and two dependent children, and its low annual income satisfies the modeled financial test, resulting in $6,064.96." -us,scenario_100,tanf,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model invented receipt of SSI even though unlisted benefit receipt must be zero. It then used that nonexistent SSI to eliminate TANF, whereas the listed facts produce a $6,064.96 TANF benefit." -us,scenario_100,tanf,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a generic $200-plus-50% earned-income disregard and an unstated payment standard. Montana's 2026 TANF parameters produce $6,064.96, not $5,298." -us,scenario_100,tanf,gemini-3.6-flash,llm_error,thresholds_rates,False,"The answer reduced the grant using unspecified earned-income disregards that do not reproduce Montana's 2026 benefit formula. Applying the actual state parameters yields $6,064.96 rather than $5,298.48." -us,scenario_100,tanf,glm-5.2,llm_error,categorical_eligibility,False,"The model double-counted the separately listed FLSA overtime premium by adding it to annual gross wages, improperly treated the excluded vehicle as a potential countable resource, and invoked a work-hours failure despite 55 weekly hours. With $5,915 of wages and the applicable Montana rules, the household receives $6,064.96." -us,scenario_100,tanf,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated absence of an explicitly listed TANF payment as proof that no benefit was payable, contrary to the instruction to calculate benefits and assume take-up. The household meets the modeled Montana eligibility conditions and receives $6,064.96." -us,scenario_100,tanf,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model overlooked the qualifying circumstances explicitly supplied: a caretaker parent, two dependent children, Montana residence, and very low earnings. Those facts trigger the TANF eligibility and payment calculation, which yields $6,064.96." -us,scenario_100,tanf,gpt-5.5,llm_error,thresholds_rates,False,"The model estimated a $438 monthly grant from an unsupported $150 countable-income figure and an unstated payment standard. Montana's 2026 income treatment and three-person benefit formula yield about $505.41 per month, or $6,064.96 annually." -us,scenario_100,tanf,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model reduced the benefit to an estimated $359.50 per month without applying the applicable Montana payment standard and earned-income rules. The correct monthly calculation annualizes to $6,064.96." -us,scenario_100,tanf,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used the wrong earned-income disregard and payment-standard combination, matching the erroneous $588-minus-$219.69 shortcut. Montana's 2026 formula yields $6,064.96." -us,scenario_100,tanf,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model invented countable SSI despite the instruction that unlisted benefit receipt is zero. With no SSI input and only $5,915 in annual wages, Montana's TANF calculation produces $6,064.96." -us,scenario_100,tanf,grok-4.3,llm_error,categorical_eligibility,False,"The model equated the absence of a listed TANF amount with a zero calculated benefit. It failed to run the eligibility and benefit logic for the low-income parent and two children, which yields $6,064.96." -us,scenario_100,tanf,grok-4.5,llm_error,thresholds_rates,False,"The model applied a $588 monthly standard and counted 75% of earnings above $200, producing about $368 monthly. Those are not the applicable Montana 2026 TANF parameters, which yield $6,064.96 annually." -us,scenario_100,tanf,grok-build-0.1,llm_error,thresholds_rates,False,"The model paired an unsupported $675 monthly maximum with the wrong $200-plus-25% disregard. That parameter combination understates the Montana 2026 result, which is $6,064.96." -us,scenario_100,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no TANF value or explanation, so it failed the required output contract." -us,scenario_100,tanf,kimi-k3,llm_error,thresholds_rates,False,"The resource treatment was consistent with eligibility, but the model then used an incorrect $588 monthly grant and counted 75% of earnings above $200. Montana's applicable 2026 payment calculation yields $6,064.96 rather than $4,419.75." -us,scenario_100,tanf,minimax-m3,llm_error,categorical_eligibility,False,"The model acknowledged that the household qualifies and that a nonzero benefit follows, then arbitrarily replaced that result with zero based on unspecified time limits and work requirements. No supplied fact triggers such a denial, and the modeled Montana benefit is $6,064.96." -us,scenario_100,tanf,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted the $710 FLSA overtime premium by adding it to the stated annual gross wages, even though gross wages already include overtime pay. It also treated proximity to a guessed gross-income limit as eliminating the grant instead of applying Montana's benefit formula to $5,915, which yields $6,064.96." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted $1,400 of personal auto-loan interest in addition to the $16,100 standard deduction. Its submitted $12,064 also contradicts both tax computations in its explanation, which produced $13,111 and $13,058 instead." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $700 of veterans benefits in gross income and used a stale $14,600 standard deduction instead of $16,100. It then misapplied the brackets and submitted $11,435 despite its own stated bracket arithmetic yielding a different amount." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model explicitly derived the exact $85,030 taxable income and $13,418.60 tax, then submitted $14,716. The failure is an unsupported final-answer substitution that contradicts its completed calculation." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's stated AGI, standard deduction, taxable income, brackets, and arithmetic yield approximately $13,419. It replaced that result with $13,876 without any computation supporting the submitted value." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used estimated standard deductions and bracket thresholds instead of the applicable 2026 values. The correct $16,100 standard deduction leaves $85,030 taxable income, which produces $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly obtained AGI of $101,130 and taxable income of $85,030, but used incorrect bracket thresholds. Its own displayed arithmetic produced $13,523.60, after which it unsupportedly changed the answer to $13,322." -us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model treated $3,840 of listed health premiums as pre-tax wage reductions even though the prompt did not identify employer-plan salary reductions. It also used a $15,400 standard deduction rather than $16,100, understating AGI, taxable income, and tax." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA individual provisions expired for 2026, reinstating a personal exemption and the 15% and 25% brackets. It also deducted the listed ESI premium from wages without a stated pre-tax contribution." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model ignored the $1,870 capital-loss offset and used a $15,300 standard deduction, producing $87,700 rather than $85,030 of taxable income. The applicable deduction is $16,100 after AGI is reduced to $101,130." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model identified AGI of $101,130 but never applied the specific $16,100 standard deduction or the applicable 2026 brackets. Those steps yield $85,030 of taxable income and $13,418.60 of tax, not the unsupported rounded estimate of $15,400." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly applied post-TCJA-expiration rules, including a personal exemption and reverted 10%, 15%, and 25% brackets. The applicable 2026 calculation uses the $16,100 standard deduction and current 10%, 12%, and 22% bracket structure at this income." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly reinstated a $5,300 personal exemption and applied pre-TCJA 15% and 25% rates. The applicable $16,100 standard deduction and 2026 brackets instead produce taxable income of $85,030 and tax of $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $15,450 standard deduction and estimated bracket thresholds. Replacing them with the applicable $16,100 deduction and 2026 thresholds yields $85,030 of taxable income and $13,418.60 of tax." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly described taxable pension income even though the supplied pension was explicitly tax-exempt. It also failed to show the $16,100 standard deduction and applicable bracket computation, resulting in an unsupported $14,498 estimate." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked unspecified itemized medical effects even though itemized deductions total only $1,871.33 and the $16,100 standard deduction controls. Its $9,587 answer reflects deductions or credits that do not apply to this filer." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly assumed scheduled expiration rules, reinstated a personal exemption, and calculated about $87,380 of taxable income. The applicable 2026 rules allow a $16,100 standard deduction without that exemption, leaving $85,030 taxable and taxed under the current bracket structure." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model claimed a phased-down qualified auto-loan-interest deduction that is not included in the reference tax base. Taxable income is therefore $85,030 after only the capital-loss adjustment and $16,100 standard deduction." -us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an estimated $15,300 standard deduction and projected brackets, producing taxable income of $85,830. The applicable $16,100 deduction leaves $85,030, and the exact 2026 bracket calculation yields $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a post-TCJA sunset, reinstating a personal exemption and the 15% and 25% brackets. The applicable calculation uses the $16,100 standard deduction and current 2026 bracket structure." -us,scenario_101,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly applied reverted TCJA law, including an $8,300 standard deduction, a $5,300 personal exemption, and 15% and 25% rates. The applicable rules instead produce $85,030 of taxable income and $13,418.60 of tax." -us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. This is a missing-output contract failure rather than a substantive tax computation. -us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted $1,174 of auto-loan interest from AGI. Without that deduction, the $16,100 standard deduction leaves taxable income of $85,030 and tax of $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model set pre-refundable-credit tax to zero by invoking CTC, EITC, and refundable mechanisms despite this single filer having no dependents and the output being measured before refundable credits. No applicable nonrefundable credit erases the $13,418.60 tax liability." -us,scenario_101,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model used estimated deduction and bracket values and calculated a preliminary tax of $13,641, then submitted $14,827 without supporting arithmetic. It also treated health premiums as pre-tax despite no such payroll treatment being specified." +us,scenario_100,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly deducted $1,128 of health premiums from FICA wages, including double-counting the overlapping $564 health-premium inputs. Those fields do not establish a pre-tax salary reduction, so Social Security and Medicare apply to the eligible wage base and total $452.48." +us,scenario_100,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated employer-sponsored insurance premiums as a pre-tax wage deduction that eliminated the entire FICA base. Employer coverage and listed premium amounts do not by themselves reduce Social Security or Medicare wages, so the payroll tax is $452.48 rather than zero." +us,scenario_100,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model reduced taxable FICA wages to zero using the employer-sponsored insurance premium input. No pre-tax Section 125 salary reduction was specified, so the eligible wages remain subject to employee Social Security and Medicare taxes totaling $452.48." +us,scenario_100,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model identified the correct 6.2% Social Security and 1.45% Medicare components but miscomputed their combined amount: 7.65% of $5,915 is about $452.50, not $453.55. PolicyEngine’s internally computed components are $366.71 and $85.76, which total $452.48." +us,scenario_100,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model’s own component calculation produced approximately $452.91, then replaced it with $477 under an invalid claim about benchmark rounding. Annual rounding cannot turn a roughly $452.5 FICA liability into $477; the computed Social Security and Medicare components total $452.48." +us,scenario_100,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below 185% of the poverty guideline as sufficient for reduced-price meals and ignored that income at or below 130% qualifies for free meals. At 44% of the guideline, reinforced by categorical eligibility, the household receives the mutually exclusive FREE tier rather than reduced-price support." +us,scenario_100,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly asserted that $5,915 was above 130% of the poverty guideline for a four-person household, even though the engine ratio is 44%. It therefore placed the household in the 130%-185% reduced-price band instead of the FREE tier, while also overlooking categorical eligibility for free meals." +us,scenario_100,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model treated wages as the household's entire countable gross income and substituted a medical-expense deduction for the $453.64 excess-shelter deduction. The correct deduction sequence leaves $237.09 of monthly net income and a $71.10 contribution, not its $89 net-income estimate." +us,scenario_100,snap,claude-haiku-4.5,llm_error,other,False,"The model established eligibility but never calculated an allotment from maximum benefits minus 30% of net income; its $1,908 figure has no derivation from the stated SNAP formula. Applying the traced monthly allotments and contributions across all 12 months yields $8,625.89." +us,scenario_100,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model counted only wages, deducted medical expenses, and then abandoned its own $9,108 calculation for an unsupported $7,320 estimate. The traced calculation instead uses $998.31 gross income and the $453.64 excess-shelter deduction, leaving $237.09 net income." +us,scenario_100,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model treated wages as total gross income and estimated that medical deductions drove net income close to zero. The applicable trace instead starts from $998.31 monthly gross income and uses the standard, earned-income, and excess-shelter deductions to reach $237.09 net income." +us,scenario_100,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used only annual wages to characterize net income and omitted the traced $998.31 monthly gross-income base and $453.64 excess-shelter deduction. Those inputs produce a $71.10 monthly contribution and the annual $8,625.89 allotment." +us,scenario_100,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model calculated gross income from wages alone, applied a medical deduction, and expressly set shelter expenses to zero. The engine applies a $453.64 excess-shelter deduction to $998.31 of gross income, producing $237.09 net income rather than $72.67." +us,scenario_100,snap,claude-sonnet-5,llm_error,period_annualization,False,"The model asserted a roughly $766 monthly maximum but annualized it to $13,700, which is arithmetically inconsistent because $766 times 12 is $9,192. It also incorrectly reduced net income to zero instead of applying the traced $71.10 monthly contribution." +us,scenario_100,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,The model included TANF-related income but used the wrong monthly amount and replaced the $453.64 excess-shelter deduction with a $46.67 medical deduction. This overstated net income at $602.21 instead of $237.09 and therefore understated SNAP. +us,scenario_100,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used wages as the full gross-income base and applied a disabled-member medical deduction, yielding only $54 of net income. The traced gross income is $998.31 and the decisive additional deduction is $453.64 of excess shelter, leaving $237.09 net income." +us,scenario_100,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer equals 12 times the $784 maximum allotment and therefore treats the household as receiving the full maximum every month. SNAP subtracts a $71.10 monthly contribution, and the maximum also rises to $802.60 in later months rather than remaining fixed." +us,scenario_100,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model described a near-maximum benefit but used an annual amount equivalent to only about $511 per month. The applicable maxima are $785 and later $802.60, reduced by only the $71.10 monthly household contribution." +us,scenario_100,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorporated TANF as income but failed to apply the traced $453.64 excess-shelter deduction correctly. The complete deductions total $761.21 and reduce $998.31 of gross income to $237.09, supporting $8,625.89 annually." +us,scenario_100,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model based the calculation on $492.92 of monthly wages and unspecified standard and medical deductions. The traced computation uses $998.31 of gross income and a $453.64 excess-shelter deduction, producing a larger $71.10 contribution than its answer reflects." +us,scenario_100,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the employer-sponsored insurance premium and all listed health costs as deductible SNAP medical expenses, reducing net income to zero. The traced deductions are the $209 standard, $98.58 earned-income, and $453.64 excess-shelter deductions, which leave $237.09 net income." +us,scenario_100,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model assigned an unsupported annual maximum of $12,588, equivalent to $1,049 per month. The three-person maxima are $785 and later $802.60, and SNAP further subtracts the $71.10 monthly contribution." +us,scenario_100,snap,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly made explicit rent or utility inputs a prerequisite for any SNAP award. The household passes the eligibility tests, receives a $453.64 excess-shelter deduction in the engine trace, and has positive monthly allotments totaling $8,625.89." +us,scenario_100,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted reported medical and insurance expenses until net income reached zero and then awarded a fixed full maximum. The trace leaves $237.09 of net income after the standard, earned-income, and excess-shelter deductions, requiring a $71.10 monthly contribution." +us,scenario_100,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,The model awarded the full $785 monthly maximum and omitted the 30% contribution based on $237.09 of net income. It also failed to account for the later-month maximum increasing to $802.60. +us,scenario_100,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly used medical deductions to reduce SNAP net income to zero and awarded the full $785 maximum. The traced deductions leave $237.09 net income, so $71.10 must be subtracted from each month's applicable maximum." +us,scenario_100,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model invoked SSI and medical deductions without deriving the SNAP income budget from the listed facts. The trace uses $998.31 gross income less $761.21 in standard, earned-income, and excess-shelter deductions, producing benefits far above $6,400." +us,scenario_100,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model returned zero without applying categorical eligibility through TANF non-cash status or the gross-income, net-income, and asset tests. Those tests are satisfied, and the monthly allotments are positive throughout the year." +us,scenario_100,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used wages alone as gross income, applied an annualized medical deduction, and did not apply the traced $453.64 monthly excess-shelter deduction. The correct monthly budget leaves $237.09 net income and a $71.10 contribution." +us,scenario_100,snap,grok-build-0.1,llm_error,period_annualization,False,"The model returned a single monthly maximum of $805 as the requested annual amount. The benchmark requires summing 12 monthly benefits after subtracting the $71.10 contribution, including later months with a $802.60 maximum." +us,scenario_100,snap,inkling,llm_error,taxable_income_or_deductions,False,"The model counted only wages and used a disabled-member medical deduction to estimate net income near $50. The traced income budget starts at $998.31 and uses a $453.64 excess-shelter deduction, leaving $237.09 net income." +us,scenario_100,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so the required output was missing." +us,scenario_100,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated premiums and medical expenses as reducing net SNAP income to zero and awarded the full $785 monthly maximum. The traced deductions leave $237.09 net income and require a $71.10 contribution, while later months use a $802.60 maximum." +us,scenario_100,snap,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model stopped after the 20% earned-income deduction, treated roughly $393 as final net income, and then used an unexplained reduction to $377 per month. It omitted the $209 standard deduction and $453.64 excess-shelter deduction that bring traced net income to $237.09." +us,scenario_100,snap,qwen-3.7-max,llm_error,period_annualization,False,"The model double-counted the $710 overtime premium by adding it to annual gross wages even though gross wages already include overtime. It also claimed a $766 monthly maximum but annualized to $12,444, although $766 times 12 is only $9,192 before the household contribution." +us,scenario_100,snap,qwen3.8-max,llm_error,asset_resource,False,"The model incorrectly disqualified the household for excess resources. The traced SNAP resource amount is $2,800, which is within the applicable limit, and categorical eligibility is also satisfied through TANF non-cash status." +us,scenario_100,ssi,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the head’s generic `is disabled` fact as satisfying SSI’s program-specific aged, blind, or disabled criterion, even though `is_ssi_aged_blind_disabled=False`. It therefore performed an inapplicable earned-income calculation instead of assigning $0 to every categorically ineligible individual; its submitted $8,285 also contradicts every amount derived in its explanation." +us,scenario_100,ssi,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model assumed that the head’s generic disability status established SSI categorical eligibility. Because the head and both children have `is_ssi_aged_blind_disabled=False`, no earned-income reduction calculation applies and each person’s SSI is $0." +us,scenario_100,ssi,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the head as an SSI-eligible disabled adult and subtracted countable earnings from a federal benefit maximum. The head fails the program-specific aged, blind, or disabled test, as do both children, so the maximum benefit and income exclusions never enter the computation." +us,scenario_100,ssi,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model explicitly classified the head as SSI eligible from the generic disability fact. SSI categorical eligibility is false for the head and both children, so no federal maximum or earned-income exclusion produces a payable benefit." +us,scenario_100,ssi,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model assumed the head qualified for SSI and calculated a federal maximum less countable earned income. The head’s `is_ssi_aged_blind_disabled` status is false, so SSI stops at categorical ineligibility and equals $0 before income counting." +us,scenario_100,ssi,grok-build-0.1,llm_error,categorical_eligibility,False,"The model equated the generic disability input with SSI disability eligibility and applied the $20 and $65 income exclusions plus the one-half earned-income rule. Those exclusions are irrelevant because no household member satisfies SSI’s aged, blind, or disabled criterion." +us,scenario_100,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable SSI value or explanation. The required output was a numeric value of 0 supported by the fact that all three individuals fail SSI’s categorical eligibility test. +us,scenario_100,ssi,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated the head’s generic disability fact as SSI categorical qualification and proceeded to income exclusions and an unsupported IRWE adjustment. The head has `is_ssi_aged_blind_disabled=False`, so SSI is $0 before any earnings, overtime, exclusions, or work-expense adjustments are considered." +us,scenario_100,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model applied a 10% Montana EITC rate instead of 20% and incorrectly reduced EITC earned income by the desired traditional 401(k) contribution. The applicable federal EITC is $2,365.89, so Montana's refundable credit is $473.18." +us,scenario_100,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that Montana has no refundable EITC. Montana refunds 20% of the household's $2,365.89 federal EITC, yielding $473.18 regardless of whether income falls below the ordinary filing threshold." +us,scenario_100,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model correctly reached a federal EITC near $2,366 but applied an obsolete 10% Montana rate instead of 20%, then submitted $297 despite its own $237 calculation. Applying 20% to $2,365.89 yields $473.18." +us,scenario_100,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model acknowledged a positive federal EITC and Montana's percentage credit but discarded that computation to submit zero. Montana's refundable rate is 20% of the $2,365.89 federal EITC, producing $473.18." +us,scenario_100,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model incorrectly classified Montana's EITC as nonrefundable and used a 10% rate. For 2026 it is refundable at 20% of the $2,365.89 federal EITC, yielding $473.18 even with no state tax liability." +us,scenario_100,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model computed the federal EITC essentially correctly but applied a 10% Montana rate instead of the applicable 20% rate. Twenty percent of $2,365.89 is $473.18." +us,scenario_100,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly treated Montana's EITC as nonrefundable and stated that no refundable credit applied. Montana refunds 20% of the $2,365.89 federal EITC, producing $473.18." +us,scenario_100,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model used both the wrong Montana rate, 10% rather than 20%, and an unsupported federal EITC of $371 rather than $2,365.89. The correct state credit is $473.18." +us,scenario_100,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly derived a federal EITC near $2,366 but applied a 10% Montana rate instead of 20%. The resulting refundable Montana EITC is $473.18." +us,scenario_100,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model omitted Montana's refundable EITC. This household's $2,365.89 federal EITC generates a Montana credit of 20%, or $473.18." +us,scenario_100,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model falsely treated the household as having zero earned income and denied the existence of an applicable refundable Montana EITC. The listed $5,915 of wages produces a $2,365.89 federal EITC and a $473.18 Montana credit at 20%." +us,scenario_100,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model incorrectly set the federal EITC to zero despite $5,915 of earned income and two qualifying children. The federal phase-in produces $2,365.89, of which Montana refunds 20%, or $473.18." +us,scenario_100,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model incorrectly asserted that Montana has no individual income tax and consequently omitted its refundable EITC. Montana's 20% supplement to the $2,365.89 federal EITC equals $473.18." +us,scenario_100,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model omitted the refundable Montana EITC triggered by the household's federal EITC eligibility. Twenty percent of the $2,365.89 federal credit yields $473.18." +us,scenario_100,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model submitted zero without applying Montana's refundable EITC formula. The household receives 20% of its $2,365.89 federal EITC, equal to $473.18." +us,scenario_100,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model applied a 10% Montana rate instead of 20% and overstated the federal EITC as $4,207 rather than $2,365.89. The correct Montana refundable EITC is $473.18." +us,scenario_100,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model correctly estimated the federal EITC at about $2,366 but used a 10% Montana rate instead of 20%. The correct state refundable credit is $473.18." +us,scenario_100,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model applied a 10% rather than 20% Montana EITC rate and added a $1,200 young-child credit that is not part of this household's refundable Montana credits. The sole component is the $473.18 Montana EITC." +us,scenario_100,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model applied a 10% rather than 20% Montana EITC rate and incorrectly added a $1,200 young-child credit for the five-year-old. Montana refundable credits here consist only of the $473.18 EITC." +us,scenario_100,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model omitted Montana's refundable EITC. The household's $2,365.89 federal EITC produces a 20% state credit of $473.18." +us,scenario_100,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that Montana offers no state EITC. Montana provides a refundable credit equal to 20% of the household's $2,365.89 federal EITC, yielding $473.18." +us,scenario_100,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model failed to apply Montana's refundable EITC. At 20% of the $2,365.89 federal EITC, the state refundable credit is $473.18." +us,scenario_100,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model used an unsupported federal EITC near $750, applied a 10% rather than 20% Montana rate, and added a $1,200 child credit that does not apply in this output. The only refundable state credit is 20% of $2,365.89, or $473.18." +us,scenario_100,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model incorrectly denied that Montana offers a refundable state EITC and focused only on the absence of facts for a property-tax circuit breaker. The refundable EITC requires no rent or property-tax input and equals 20% of $2,365.89, or $473.18." +us,scenario_100,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model correctly computed the federal EITC near $2,366 but applied a 10% Montana rate instead of 20%. The correct refundable Montana EITC is $473.18." +us,scenario_100,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model incorrectly asserted that Montana has no state income tax and therefore omitted the state EITC. Montana refunds 20% of the $2,365.89 federal EITC, equal to $473.18." +us,scenario_100,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly asserted that Montana has no EITC and treated zero state liability as preventing a credit. Montana's EITC is refundable at 20% of the $2,365.89 federal EITC, yielding $473.18." +us,scenario_100,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model omitted Montana's refundable EITC. The household qualifies for a $2,365.89 federal EITC, and Montana refunds 20% of it, producing $473.18." +us,scenario_100,tanf,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly found countable monthly earnings below its own payment standard, then contradicted that calculation by declaring the benefit zero. Montana’s eligibility and payment calculation instead yields $6,064.96 annually." +us,scenario_100,tanf,claude-haiku-4.5,llm_error,asset_resource,False,"The model used an incorrect $2,000 resource limit and incorrectly described the assistance unit as four people. The $2,800 bank balance does not disqualify this three-person Montana TANF unit, whose benefit is $6,064.96." +us,scenario_100,tanf,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model converted general concerns about income, assets, and work participation into ineligibility without applying Montana’s actual tests. The low-income parent-and-two-child unit passes those tests and receives $6,064.96." +us,scenario_100,tanf,claude-opus-4.8,llm_error,thresholds_rates,False,"The model approximated a $595 monthly award from an estimated maximum grant and disregards instead of applying Montana’s 2026 benefit formula. That formula produces $6,064.96 annually, not $7,140." +us,scenario_100,tanf,claude-opus-5,llm_error,categorical_eligibility,False,"The unexplained zero treats the household as ineligible despite its income and family composition satisfying Montana’s TANF rules. Applying the state calculation yields $6,064.96." +us,scenario_100,tanf,claude-sonnet-4.6,llm_error,asset_resource,False,"The model imposed an incorrect $1,000 liquid-asset limit and therefore rejected the household solely because of its $2,800 bank balance. Montana’s modeled resource rules do not disqualify the unit, and the benefit is $6,064.96." +us,scenario_100,tanf,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used an incorrect $327 monthly payment standard and a generic 20% plus $200 disregard. Montana’s 2026 payment parameters and benefit formula yield $6,064.96 annually." +us,scenario_100,tanf,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model substituted a $605 maximum grant and a $200-plus-50% earnings disregard for Montana’s applicable 2026 parameters. Its resulting $458.54 monthly estimate understates the computed annual benefit of $6,064.96." +us,scenario_100,tanf,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $588 payment standard and treated 75% of earnings above $200 as countable, then introduced an unsupported rounding adjustment. Montana’s actual 2026 formula produces $6,064.96." +us,scenario_100,tanf,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model asserted failure of unspecified state requirements without applying the income and family-composition tests. The household qualifies under Montana’s rules and receives $6,064.96." +us,scenario_100,tanf,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated SSI receipt as eliminating TANF even though no SSI receipt was supplied and the benchmark’s separate SSI output does not establish such income. The listed facts produce a Montana TANF benefit of $6,064.96." +us,scenario_100,tanf,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied an unsupported $200-plus-50% earned-income disregard and an approximate Pathways grant calculation. Montana’s applicable 2026 formula yields $6,064.96 rather than $5,298." +us,scenario_100,tanf,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model’s $5,298.48 answer embodies the wrong earned-income-disregard or payment-standard parameters. Applying Montana’s 2026 TANF calculation gives $6,064.96." +us,scenario_100,tanf,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model double-counted the $710 FLSA overtime premium by adding it to the stated annual gross wages, even though gross wages already include overtime pay. It also treated excluded vehicle value and work participation as barriers; the correct Montana calculation awards $6,064.96." +us,scenario_100,tanf,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly listed TANF payment as proof that none was payable, ignoring the instruction to calculate benefits with take-up assumed. The household’s facts establish eligibility and a $6,064.96 annual award." +us,scenario_100,tanf,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model demanded a separate TANF trigger even though two dependent children, a caretaker parent, Montana residence, and low income supply the eligibility facts. Applying the program rules yields $6,064.96." +us,scenario_100,tanf,gpt-5.5,llm_error,thresholds_rates,False,"The model estimated a $588 payment standard and roughly $150 of countable monthly earnings instead of using Montana’s exact 2026 parameters. Its shortcut yields $5,256 rather than the computed $6,064.96." +us,scenario_100,tanf,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model reduced an unspecified three-person payment standard by an estimated earnings amount to reach $359.50 per month. Montana’s exact benefit formula produces $6,064.96 annually." +us,scenario_100,tanf,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used the same $588-standard and $200-plus-25% disregard shortcut reflected in a $4,419.75 annual result. Those are not the applicable Montana 2026 computation parameters; the correct award is $6,064.96." +us,scenario_100,tanf,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model invented countable SSI income even though no SSI receipt or amount was listed and unlisted income must be zero. With only the stated income included, Montana TANF equals $6,064.96." +us,scenario_100,tanf,grok-4.3,llm_error,categorical_eligibility,False,"The model returned zero because it did not calculate TANF from the listed household facts. Those facts establish Montana eligibility and produce $6,064.96." +us,scenario_100,tanf,grok-4.5,llm_error,thresholds_rates,False,"The model used a $588 monthly payment standard and counted 75% of earnings above $200, producing about $368 monthly. Montana’s applicable 2026 parameters instead yield $6,064.96 annually." +us,scenario_100,tanf,grok-build-0.1,llm_error,thresholds_rates,False,"The model combined an unsupported $675 maximum grant with the wrong $200-plus-25% earnings-disregard calculation. Montana’s actual benefit formula yields $6,064.96, not $5,464." +us,scenario_100,tanf,inkling,llm_error,thresholds_rates,False,"The model relied on an approximate $588 maximum grant and a $200-plus-25% earnings disregard. Using Montana’s exact 2026 parameters produces $6,064.96." +us,scenario_100,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no TANF value or explanation, so the required output was missing." +us,scenario_100,tanf,kimi-k3,llm_error,thresholds_rates,False,"The model correctly kept the resources within its stated limit but used a $588 grant and counted 75% of earnings above $200. That incorrect parameterization produces $4,419.75 instead of Montana’s computed $6,064.96." +us,scenario_100,tanf,minimax-m3,llm_error,categorical_eligibility,False,"The model acknowledged that the household qualified and should receive a nonzero benefit, then replaced that conclusion with zero based on unspecified time-limit and work-rule concerns. Applying the actual eligibility and payment rules yields $6,064.96." +us,scenario_100,tanf,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted the $710 overtime premium despite the prompt stating that annual gross wages already include overtime. It then compared that inflated income with an approximate annual payment standard instead of applying the state benefit formula, which yields $6,064.96." +us,scenario_100,tanf,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly classified $5,915 of annual income for a three-person family as too high under Montana’s 2026 TANF rules. The household passes the income test and receives $6,064.96." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted $1,400 of personal auto-loan interest in addition to the standard deduction, reducing taxable income below $85,030. It then submitted $12,064 despite its own alternative calculations producing $13,111 and $13,058." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $700 of veterans benefits in gross income and used a $14,600 standard deduction instead of $16,100. It also misapplied the brackets and submitted a figure inconsistent with its stated $10,637.50 calculation." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly derived $85,030 of taxable income and explicitly calculated $13,418.60 of tax. It then replaced that result with $14,716 without any supporting computation." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly computed approximately $13,419 from the proper AGI, standard deduction, taxable income, and brackets. It then submitted $13,876, a value unsupported by its own arithmetic." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used estimated standard deductions and bracket thresholds rather than the enacted 2026 values. The correct $16,100 deduction and 2026 brackets applied to $85,030 produce $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model reached the correct $85,030 taxable income but used inaccurate bracket thresholds to calculate $13,523.60. It then submitted $13,322 based on an unexplained indexing adjustment rather than its stated arithmetic." +us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly treated $3,840 of listed health premiums as pre-tax wage deductions and used a $15,400 standard deduction. No payroll exclusion for those premiums was supplied, so AGI is $101,130 and taxable income is $85,030." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA rules expired for 2026, restoring a personal exemption and the 15% and 25% brackets. It also subtracted the listed insurance premium from wages without a stated pre-tax payroll treatment." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model ignored the $1,870 capital-loss deduction and used a $15,300 standard deduction, producing $87,700 rather than $85,030 of taxable income. Applying the enacted brackets to the correct taxable income yields $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model identified the correct $101,130 AGI but never applied the specific $16,100 standard deduction or enacted 2026 bracket thresholds. Its rounded $15,400 estimate overstates the tax on $85,030 of taxable income." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly applied post-TCJA-expiration rules, including a personal exemption and reverted 10%, 15%, and 25% brackets. The operative 2026 rules instead provide a $16,100 standard deduction and retain the 10%, 12%, and 22% brackets applicable here." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly restored a $5,300 personal exemption and pre-TCJA tax brackets for 2026. Taxable income is $85,030 under the $16,100 standard deduction, and the enacted brackets produce $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $15,450 standard deduction and estimated bracket thresholds instead of the enacted 2026 parameters. The $16,100 deduction reduces taxable income to $85,030, whose tax is $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the tax-exempt pension as taxable and described the capital loss as though positive gains remained. The pension and veterans benefits are excluded, the $1,870 loss reduces wages to $101,130 of AGI, and the resulting tax is $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model applied unspecified itemized medical effects and nonrefundable credits that the facts do not support. The standard deduction exceeds the $1,871.33 of itemized deductions, and no nonrefundable credit reduces the $13,418.60 tax." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and scheduled post-TCJA parameters, producing about $87,380 of taxable income. The operative $16,100 standard deduction produces $85,030 of taxable income taxed at $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model deducted phased-down qualified auto-loan interest even though the reference computation allows no such additional deduction. After the $16,100 standard deduction, taxable income remains $85,030 and tax is $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an estimated $15,300 standard deduction and projected brackets, overstating taxable income as $85,830. The enacted $16,100 deduction and exact 2026 thresholds yield $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a post-TCJA sunset, restored a personal exemption, and applied the former 15% and 25% brackets. The operative 2026 standard deduction and brackets produce $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly used reverted TCJA law, including a personal exemption and 10%, 15%, and 25% brackets. Under the operative rules, the $16,100 standard deduction produces $85,030 of taxable income and $13,418.60 of tax." +us,scenario_101,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly derived $85,030 of taxable income but did not apply the exact enacted bracket thresholds and arithmetic. Those brackets yield $13,418.60, not the rounded $13,426 estimate." +us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly claimed a $1,174 qualified auto-loan-interest deduction after an invented phase-down calculation. No additional auto-interest deduction reduces taxable income, which remains $85,030 after the standard deduction." +us,scenario_101,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model zeroed the liability by invoking CTC, EITC, and refundable mechanisms despite the absence of children and despite refundable credits being excluded from this output. This single filer has no applicable nonrefundable credit, so the full $13,418.60 remains before refundable credits." +us,scenario_101,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's stated bracket calculation produced $13,641, but it submitted $14,827 without any supporting adjustment. It also used estimated deduction and bracket values instead of the $16,100 standard deduction and exact enacted thresholds." +us,scenario_101,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,payroll_tax_base,False,"The model incorrectly subtracted $4,824 of employee payroll tax as a nonrefundable federal income-tax credit; employee payroll tax does not offset this liability. It also failed to deduct the $1,870 capital loss from wages and used an incorrect standard deduction and bracket computation." us,scenario_101,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_101,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model computed the correct FICA components and explicitly summed them to $7,879.50, then submitted $8,149.50 instead. Its error is an internal answer-contract mismatch: the numeric value field does not match its own stated payroll tax calculation." -us,scenario_101,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model computed Social Security tax of $6,386.00 and Medicare tax of $1,493.50 and correctly stated their total as $7,879.50, then submitted $7,854.00. Its error is an internal answer-contract mismatch: the submitted numeric value does not match the calculation in its explanation." -us,scenario_101,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model incorrectly reduced payroll-taxable wages by the $1,920 employer-sponsored health insurance premium, using $101,080 instead of the full $103,000 of W-2 wages. PolicyEngine's employee payroll tax calculation applies Social Security and Medicare rates to the wage input here, with no reduction for the listed health insurance premium." -us,scenario_101,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly computed employee Social Security tax of $6,386 and Medicare tax of $1,493.50, with no Additional Medicare Tax and no Texas employee payroll tax, yielding $7,879.50. It then replaced that result with an unsupported $7,757 submission under a false appeal to PolicyEngine rounding." -us,scenario_101,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable payroll_tax value or explanation. This is a missing-output failure rather than a substantive payroll tax calculation. -us,scenario_101,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated employee Social Security and Medicare withholding as outside the reported payroll tax liability and returned only Texas state payroll tax, which is zero. The requested output includes employee-side federal FICA, so the $103,000 of W-2 wages generates $6,386 of Social Security tax and $1,493.50 of Medicare tax." -us,scenario_101,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model invented a Texas state income tax calculation on taxable income after deductions and nonrefundable credits. Texas has no state individual income tax, so no taxable-income computation can produce a Texas state income tax liability for this output." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model inconsistently considered subtracting the $7,989 employer-sponsored insurance premium from the stated wages and then submitted $5,459 even though its own no-premium calculation was about $6,092. The supplied wages remain in gross income, and the $32,200 standard deduction plus $0.20 QBI deduction yields $55,000.80 of taxable income and $6,104.10 of tax." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model incorrectly treated estimated wage withholding as a subtraction from federal income tax before refundable credits. Withholding is a payment against tax, not a credit in this output; the tax liability is $6,104.10 before refundable credits." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used estimated 2026 deduction and bracket values, calculated approximately $6,204 from them, and then submitted the arithmetically unsupported $5,764. The applicable $32,200 standard deduction, $0.20 QBI deduction, and 2026 rate schedule produce $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's stated computation produced about $6,092, but it submitted $6,633 with no calculation supporting that amount. Applying the exact deduction and bracket parameters to $55,000.80 of taxable income yields $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly reconstructed approximately $6,104 from the income, standard deduction, and brackets, then replaced that result with an unsupported $5,983. The unrounded engine computation is $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model estimated the standard deduction as $30,800 instead of applying the $32,200 amount and omitted the $0.20 QBI deduction. This overstated taxable income and produced $6,278 instead of $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's own bracket calculation produced about $6,056, but it submitted $6,800 and incorrectly attributed a large difference to negligible self-employment adjustments. The $1 of business income creates a $0.20 QBI deduction, and the exact tax is $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $30,000 standard deduction instead of $32,200 and omitted the $0.20 QBI deduction. It therefore taxed $57,201 rather than $55,000.80 and overstated the liability." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a pre-TCJA-style $16,000 standard deduction, personal exemptions, and a 15% bracket instead of the operative 2026 standard deduction and rate schedule. It also removed the $7,989 premium from the stated wages, so neither its taxable-income base nor its rates match the required computation." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model claimed to apply the 2026 brackets to $57,201 of taxable income but submitted $4,124, which those brackets do not produce. It also used a $30,000 standard deduction rather than $32,200 and omitted the $0.20 QBI deduction." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset and applied personal exemptions and pre-TCJA brackets. It also subtracted the $7,989 employer-sponsored premium from wages even though the stated wage amounts enter gross income; the operative rules yield $55,000.80 of taxable income and $6,104.10 of tax." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly used post-sunset parameters, including a $16,000 standard deduction and a 15% marginal bracket, and separately deducted the $7,989 premium from stated wages. The operative 2026 rules use the $32,200 joint standard deduction and 10%/12% schedule at this income." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $7,989 employer-sponsored insurance premium from the wage amounts and used a $30,800 standard deduction. Gross income remains $87,201, and the $32,200 standard deduction plus $0.20 QBI deduction produces $55,000.80 of taxable income." -us,scenario_102,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model explicitly calculated $6,398.12 but submitted $1,737, an unsupported output unrelated to its reasoning. Using the exact $32,200 standard deduction, $0.20 QBI deduction, and 2026 brackets produces $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The submitted $3,357 does not result from the joint rate schedule applied after the available deductions. The correct taxable income is $55,000.80, and the 2026 brackets produce $6,104.10 with no nonrefundable credit reduction." -us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model incorrectly reduced the income tax to zero despite identifying no applicable nonrefundable credit. The standard and QBI deductions leave $55,000.80 taxable, which generates $6,104.10 of tax." -us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model separately deducted the $7,989 employer-sponsored insurance premium from the supplied wage income. The stated wages remain in the $87,201 gross-income base, leaving $55,000.80 taxable after the standard and QBI deductions and producing $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model separately deducted the $7,989 employer-sponsored insurance premium from the supplied wages and invoked a half-self-employment-tax deduction despite the de minimis $1 business income. The correct deductions are the $32,200 standard deduction and $0.20 QBI deduction, yielding $6,104.10 of tax." -us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $57,200, reflecting an understated standard deduction and omission of the $0.20 QBI deduction. The correct taxable income is $55,000.80, and the exact 2026 schedule yields $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model estimated the standard deduction as $30,840 instead of using $32,200 and omitted the $0.20 QBI deduction. Its resulting $56,361 taxable-income figure was too high; the correct figure is $55,000.80." -us,scenario_102,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $7,989 employer-sponsored insurance premium from the stated wages and used a $30,800 standard deduction. The supplied wages plus self-employment income total $87,201, and the applicable deductions leave $55,000.80 taxable." +us,scenario_101,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly derived $6,386 of Social Security tax and $1,493.50 of Medicare tax, then submitted $8,149.50 despite its own arithmetic totaling $7,879.50. Its unexplained $270 addition is not produced by a wage-base cap, rounding, Additional Medicare Tax, or Texas employee payroll tax." +us,scenario_101,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly calculated total employee payroll tax as $7,879.50 but submitted $7,854.00. The $25.50 reduction has no basis in any payroll-tax component stated in its reasoning." +us,scenario_101,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly deducted the $1,920 health-insurance premium from FICA wages. The prompt identifies premiums paid but does not establish a pre-tax employer-plan salary reduction, so both Social Security and Medicare taxes apply to the full $103,000 of wages." +us,scenario_101,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly computed $7,879.50, then replaced it with $7,757 using a nonexistent PolicyEngine rounding adjustment. Texas adds no mandatory employee payroll tax here, and rounding cannot reduce the correctly summed components by $122.50." +us,scenario_101,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required output contract." +us,scenario_101,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated ordinary withholding of employee FICA taxes as though it eliminated annual payroll-tax liability. W-2 wages of $103,000 generate $6,386 of employee Social Security tax and $1,493.50 of employee Medicare tax regardless of employer-sponsored insurance or Texas's lack of a state payroll tax." +us,scenario_101,payroll_tax,qwen3.8-max,llm_error,other,False,"The model correctly summed Social Security and Medicare taxes to $7,879.50, then added an unexplained $152. No Additional Medicare Tax or mandatory Texas employee payroll tax applies, so that extra amount has no payroll-tax source." +us,scenario_101,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model invented a Texas individual income tax liability by applying an unstated tax rate to taxable income. Texas imposes no individual state income tax, so the correct state income tax before refundable credits is $0, not $6,035." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model's own no-ESI calculation produced tax near $6,092, but it submitted $5,459 without a rate-schedule computation supporting that figure. It also used a $32,300 standard deduction instead of $32,200 and omitted the $0.20 QBI deduction." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model incorrectly subtracted invented federal withholding from income tax liability. Withholding is a payment, not a nonrefundable credit, so the requested pre-refundable-credit tax remains the tax computed from $55,000.80 of taxable income." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an incorrect $31,500 standard deduction and an inaccurate 2026 bracket threshold, then submitted $5,764 even though its displayed bracket arithmetic produced about $6,204. The correct deductions leave $55,000.80 taxable and the rate schedule yields $6,104.10." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly derived tax near $6,092 from its approximate inputs but submitted $6,633, a number unsupported by its calculation. Using the exact $32,200 standard deduction, $0.20 QBI deduction, and 2026 schedule gives $6,104.10." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's stated calculation reached about $6,104, but it replaced that result with $5,983 without any deduction, credit, or rate computation supporting the reduction. The exact computation yields $6,104.10." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used an estimated $30,800 standard deduction instead of the 2026 $32,200 amount and omitted the $0.20 QBI deduction. This overstated taxable income as $56,401 instead of $55,000.80 and therefore overstated the tax." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's displayed bracket calculation produced about $6,056, but it submitted $6,800 and incorrectly attributed the $744 increase to negligible self-employment effects. The $1 of business income instead creates a $0.20 QBI deduction, and the exact tax is $6,104.10." +us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used the 2025 $30,000 joint standard deduction and 2025 bracket threshold instead of the 2026 values. The $32,200 standard deduction plus $0.20 QBI deduction reduces taxable income to $55,000.80." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a fictional post-sunset regime with a $16,000 standard deduction, personal exemptions, and a 15% bracket. It also subtracted the listed $7,989 ESI premium from wages even though the stated wage amounts feed gross income directly; the applicable 2026 computation uses the $32,200 standard deduction and 10%/12% schedule." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model claimed $57,201 of taxable income under a $30,000 deduction but submitted $4,124, which does not follow from the 2026 joint rate schedule. The exact deductions produce $55,000.80 taxable and $6,104.10 of tax." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset, introduced personal exemptions, used pre-TCJA brackets, and separately removed $7,989 of ESI premiums from the stated wages. The applicable 2026 rules use a $32,200 joint standard deduction, no personal exemptions, and the extended 10%/12% schedule." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly used a post-TCJA-expiration $16,000 standard deduction and a 15% marginal bracket, while also subtracting $7,989 of ESI premiums from the stated wages. The 2026 joint-filer computation instead uses the $32,200 standard deduction and the 10%/12% brackets." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted the spouse's $7,989 ESI premium from the explicitly stated wage income and used a $30,800 standard deduction. Gross income remains $87,201, and the $32,200 standard deduction plus $0.20 QBI deduction leaves $55,000.80 taxable." +us,scenario_102,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model's displayed arithmetic produced $6,398.12, but it submitted $1,737 without identifying any credit or adjustment that reduces the tax. No nonrefundable credit applies, and the exact deductions and rates yield $6,104.10." +us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The submitted $3,357 does not result from applying the 2026 joint-filer schedule to the household's income after the standard and QBI deductions. Taxable income is $55,000.80, and the bracket computation yields $6,104.10 with no nonrefundable credit offset." +us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model incorrectly reduced the liability to zero despite identifying no nonrefundable credit. The standard and QBI deductions leave $55,000.80 taxable, which produces $6,104.10 under the 2026 joint-filer schedule." +us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the listed $7,989 employer-sponsored premium as an additional subtraction from the stated wages. The wages enter gross income at $87,200 as given, and only the $32,200 standard deduction and $0.20 QBI deduction reduce taxable income here." +us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the $7,989 employer-sponsored insurance premium from the stated wage amounts and invoked a half-SE-tax deduction despite the de minimis $1 business income. The applicable deductions are the $32,200 standard deduction and $0.20 QBI deduction, producing $55,000.80 taxable income." +us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used taxable income near $57,200, reflecting a $30,000 standard deduction rather than the 2026 $32,200 amount, and omitted the $0.20 QBI deduction. Correct taxable income is $55,000.80 and the resulting tax is $6,104.10." +us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $30,840 standard deduction instead of $32,200 and omitted the $0.20 QBI deduction. That overstated taxable income by $1,360.20 and consequently overstated the tax." +us,scenario_102,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted the spouse's $7,989 ESI premium from the stated $46,000 of wages and used a $30,800 standard deduction. The given wages remain in gross income, and the correct $32,200 standard deduction plus $0.20 QBI deduction produces $55,000.80 taxable income." us,scenario_102,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_102,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model estimated the joint standard deduction as $30,750 instead of applying $32,200 and omitted the $0.20 QBI deduction. This overstated taxable income by $1,450.20 and therefore overstated the tax." -us,scenario_102,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model invented a Child and Dependent Care Credit despite no listed dependent-care expenses and then applied a credit amount that does not reconcile its stated $6,324 pre-credit tax to the submitted $3,602. Disability alone does not create CDCC without qualifying care expenses; no nonrefundable credit reduces the $6,104.10 tax here." -us,scenario_102,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model invented a disability-related refundable credit and treated disability status as generating $3,995 of federal refundable credits. The credit for the elderly or disabled is not a refundable credit, the household has no qualifying children for refundable CTC, no childless EITC at this income, and no listed qualified education expenses for refundable AOTC." -us,scenario_102,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not submit a parseable value or explanation for federal_refundable_credits. The required output was $0 because every refundable-credit component evaluated for this childless, high-income household returns zero." -us,scenario_102,head_medicaid_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model correctly rejected North Carolina expansion Medicaid and SSI-related ABD income eligibility, then incorrectly invented a medically needy spend-down pathway using medical expenses and rent. PolicyEngine assigns medicaid_category = NONE for this person, so those expenses do not convert the over-income disabled adult into Medicaid eligibility." -us,scenario_102,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated disability plus $1,001 in bank assets as sufficient for North Carolina Medicaid and ignored the applicable income test. The head's household MAGI is $87,201, or 4.03 times FPL, so the MAGI adult pathway fails and no disabled-adult pathway is assigned." -us,scenario_102,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated the household’s generic disability flag as sufficient for Medicare eligibility. At age 26, the head does not satisfy the age-based Medicare rule applied by PolicyEngine, so the disability fact does not change the output from 0." -us,scenario_102,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model incorrectly asserted that PolicyEngine automatically converts disability status into Medicare eligibility for a person under 65. PolicyEngine’s computation does not use the bare disability flag as an independent qualifying pathway here, and the 26-year-old head fails the age threshold." -us,scenario_102,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model misapplied the eligibility rule by making listed disability sufficient despite the head being under 65. The computation applies the Medicare age threshold, which the 26-year-old head does not meet." -us,scenario_102,head_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model invented a simple disjunctive rule of age 65 or disabled and treated the generic disability input as satisfying Medicare eligibility. The applicable PolicyEngine computation requires the age criterion here, so the head’s age of 26 yields 0." -us,scenario_102,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model repeatedly used 0.145 instead of 0.0145 for the 1.45% Medicare rate and then submitted $7,149.68, a figure unsupported by any of its calculations. Applying 6.2% and 1.45% to $87,200 yields $6,670.80." -us,scenario_102,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model correctly derived $6,670.80 but submitted $6,672.30. Its output added an unexplained $1.50 to its own exact calculation." -us,scenario_102,payroll_tax,claude-opus-5,llm_error,other,False,"The model correctly calculated Social Security of $5,406.40 and Medicare of $1,264.40 but replaced their exact $6,670.80 sum with an incorrect rounded submission of $6,672.00." -us,scenario_102,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model listed all four correct spouse-level components and correctly summed them to $6,670.80, then submitted $6,664.60. The submitted value is $6.20 below its own derivation." -us,scenario_102,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly subtracted the spouse’s $7,989 employer-sponsored insurance premium from FICA wages. The prompt supplies gross wages of $46,000 and does not identify that premium as an employee pre-tax payroll deduction, so total FICA wages remain $87,200." -us,scenario_102,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced the spouse’s payroll-tax base from $46,000 to $38,011 by treating the listed employer-sponsored insurance premium as a pre-tax employee deduction. FICA applies here to the full listed wages, producing $6,670.80." -us,scenario_102,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly treated the $7,989 employer-sponsored insurance premium as reducing the spouse’s FICA-taxable wages. Using the full $41,200 and $46,000 wage amounts at 7.65% yields $6,670.80." -us,scenario_102,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model deducted $7,989 from the spouse’s wage base without a fact establishing an employee pre-tax payroll deduction. The spouse’s full $46,000 is subject to employee Social Security and Medicare tax." -us,scenario_102,payroll_tax,glm-5.2,llm_error,other,False,"The model correctly derived $5,406.40 of Social Security tax plus $1,264.40 of Medicare tax, totaling $6,670.80, but submitted $6,667.80. Its final output is an unexplained $3 below its own arithmetic." -us,scenario_102,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model evaluated $87,200 × 0.0765 incorrectly. That multiplication equals $6,670.80, not $7,063.00." -us,scenario_102,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model improperly reduced wage compensation by the spouse’s $7,989 employer-sponsored insurance premium. The payroll-tax base is the full $87,200 of listed wages, yielding $6,670.80." -us,scenario_102,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model treated the spouse’s employer-sponsored premium as a pre-tax employee deduction and taxed only $38,011 of the spouse’s wages. The supplied payroll-tax wage amount is the full $46,000." -us,scenario_102,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model wrongly deducted the $7,989 employer-sponsored insurance premium from the spouse’s FICA wage base and then rounded the resulting estimate. Applying 7.65% to the full $87,200 of wages gives exactly $6,670.80." +us,scenario_102,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used a $30,750 standard deduction instead of the 2026 $32,200 deduction and omitted the $0.20 QBI deduction. This overstated taxable income as $56,451 rather than $55,000.80." +us,scenario_102,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model invented a Child and Dependent Care Credit even though the household contains only two 26-year-old spouses and no qualifying child or dependent-care expenses. Disability alone does not create CDCC when neither spouse is identified as the other's dependent or as incapable of self-care with qualifying care expenses; no nonrefundable credit reduces the $6,104.10 tax." +us,scenario_102,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model invented two qualifying children and a $4,000 Child Tax Credit even though the household lists only the head and spouse. It also miscomputed pre-credit tax as $8,631; the actual joint taxable income is $55,000.80 and no CTC applies." +us,scenario_102,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a $3,995 refundable disability-related credit even after correctly ruling out EITC, refundable CTC, and the American Opportunity Credit. The Credit for the Elderly or the Disabled is nonrefundable, disability alone does not establish it for these working 26-year-olds, and no listed fact generates any refundable component." +us,scenario_102,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for federal_refundable_credits, violating the required output contract." +us,scenario_102,head_medicaid_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model invented a medically needy spend-down pathway in the PolicyEngine determination and improperly treated rent as a Medicaid spend-down expense. The head qualifies for no Medicaid category, and the household's $87,201 MAGI exceeds every applicable income threshold, so the listed expenses and assets do not change the result." +us,scenario_102,head_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model incorrectly declared $41,201 within North Carolina's income limit for a disabled adult and evaluated only the head's earnings instead of the applicable household income. The trace uses household MAGI of $87,201, equal to 4.03 times FPL, and assigns no Medicaid eligibility category." +us,scenario_102,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated disability as automatically conferring Medicare eligibility at age 26. It failed to apply the age-based eligibility rule used for this output, under which the head is below 65 and therefore not eligible." +us,scenario_102,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,The model incorrectly asserted that PolicyEngine converts the generic disabled input directly into Medicare eligibility for an adult under 65. The head's age of 26 fails the eligibility criterion applied by this variable. +us,scenario_102,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model misapplied the disability flag as an under-65 Medicare eligibility pathway. PolicyEngine's computation for this output requires the applicable age criterion, which the 26-year-old head does not meet." +us,scenario_102,head_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model used an incorrect age-or-disability rule and treated any listed disability as sufficient for Medicare. The head is 26, so the age test fails, and the generic disability fact does not establish Medicare eligibility." +us,scenario_102,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model repeatedly used 0.145 instead of 0.0145 for Medicare, incorrectly considered subtracting unlisted pretax premium contributions from FICA wages, and then submitted $7,149.68 without deriving it. Applying 6.2% and 1.45% to the full $87,200 of wages yields $6,670.80." +us,scenario_102,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model correctly computed $3,151.80 plus $3,519.00 as $6,670.80 but submitted $6,672.30. Its output is a $1.50 transcription or arithmetic error that contradicts its own derivation." +us,scenario_102,payroll_tax,claude-opus-5,llm_error,other,False,"The model correctly derived Social Security tax of $5,406.40 and Medicare tax of $1,264.40 but rounded their exact $6,670.80 sum to $6,672.00. Payroll tax is an exact annual amount here, so that unsupported approximation caused the error." +us,scenario_102,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model listed all four correct person-level components and explicitly summed them to $6,670.80, then submitted $6,664.60. The submitted value is inconsistent with its own correct arithmetic by $6.20." +us,scenario_102,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the spouse’s $7,989 employer-sponsored insurance premium from FICA-taxable wages even though the prompt does not state that this amount is an employee pretax salary reduction. Payroll tax applies to the full listed wages of $87,200, producing $6,670.80 rather than $6,059.64." +us,scenario_102,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced the spouse’s payroll-tax wage base from $46,000 to $38,011 by treating the listed employer-sponsored insurance premium as a pretax employee contribution. With no such employee deduction specified, both spouses’ full wages are subject to 7.65% FICA, totaling $6,670.80." +us,scenario_102,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly deducted $7,989 of employer-sponsored insurance premiums from the spouse’s FICA wages. The stated gross wages remain the payroll-tax base, so 7.65% applies to $87,200 and yields $6,670.80." +us,scenario_102,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model treated the $7,989 employer-sponsored insurance premium as an employee pretax payroll deduction and taxed only $38,011 of the spouse’s $46,000 wages. The full listed wages are subject to employee Social Security and Medicare taxes, yielding $6,670.80." +us,scenario_102,payroll_tax,glm-5.2,llm_error,other,False,"The model correctly calculated $5,406.40 of Social Security tax and $1,264.40 of Medicare tax and correctly stated their sum as $6,670.80, but submitted $6,667.80. The final value is a $3 transcription error." +us,scenario_102,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct formula, but its multiplication is wrong: $87,200 × 0.0765 equals $6,670.80, not $7,063.00. No other payroll-tax component accounts for the difference." +us,scenario_102,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model reduced wage compensation by the spouse’s $7,989 employer-sponsored insurance premium without a stated employee pretax contribution. Using the full $87,200 wage base gives $5,406.40 of Social Security tax and $1,264.40 of Medicare tax, totaling $6,670.80." +us,scenario_102,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model netted the spouse’s $7,989 employer-sponsored insurance premium against wages for FICA purposes. Because the prompt supplies no employee pretax salary reduction, payroll taxes apply to the full $46,000 of spouse wages and total $6,670.80 for the household." +us,scenario_102,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly removed the $7,989 employer-sponsored insurance premium from the spouse’s FICA wage base and then rounded its result. Applying 7.65% to the full $87,200 of listed wages yields exactly $6,670.80." us,scenario_102,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required output contract." -us,scenario_102,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly computed $5,406.40 of Social Security tax and $1,264.40 of Medicare tax but replaced their exact $6,670.80 sum with $6,539.00. That submitted figure is not a rounding of the stated calculation." -us,scenario_102,self_employment_tax,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model imposed self-employment tax on the listed $1 self-employment-income fact and then invented a minimum or rounding treatment that produced $31. Self-employment tax does not apply to wage income, and the reference computation contains no qualifying net earnings from self-employment activities, so the self-employment tax calculation stays at $0." -us,scenario_102,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model failed to return a parseable self_employment_tax value. The required output is $0 because the computation has no qualifying net self-employment earnings and self-employment tax is excluded from employee payroll tax. -us,scenario_102,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model invented a simplified PolicyEngine rule under which disability automatically confers Medicare eligibility regardless of age. At age 26, the spouse does not meet the age threshold, and the listed disability alone does not activate a Medicare eligibility pathway." -us,scenario_102,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model incorrectly treated the spouse's disability indicator as sufficient for Medicare eligibility. The spouse is 26, and disability without a qualifying Medicare entitlement status does not override the age-65 requirement." -us,scenario_102,spouse_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model misapplied disability as an independent under-65 Medicare eligibility rule. The household facts provide no qualifying under-65 Medicare entitlement, so the 26-year-old spouse is not eligible." -us,scenario_102,spouse_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model used an incorrect age-or-disability test and concluded that any disabled person qualifies for Medicare. Disability alone is not sufficient, and the spouse is below age 65 with no qualifying under-65 Medicare entitlement." -us,scenario_102,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly derived $61,701 of taxable income and a 3.99% tax of about $2,462, then discarded that calculation for an unsupported $2,755 based on an alternative rate and tax base. The required calculation is $61,701 × 3.99% = $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model falsely treated North Carolina as having no individual income tax in 2026. North Carolina taxes the household's $61,701 of taxable income at 3.99%, producing $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a 4.25% rate instead of North Carolina's 3.99% rate for 2026, despite correctly deriving $61,701 of taxable income. It also reported $2,587 even though its stated 4.25% computation produced about $2,622; the correct computation is $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model explicitly computed $61,701 × 3.99% as about $2,462 but submitted $3,389, a number unsupported by its reasoning. The correctly stated inputs produce $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly identified the $25,500 deduction, $61,701 taxable income, and 3.99% rate, then submitted $2,792 instead of the resulting $2,461.87. Its final value contradicts its own computation." -us,scenario_102,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $21,500 joint standard deduction instead of $25,500, overstating taxable income by $4,000, and also used 4.25% instead of 3.99%. The proper base is $61,701 and the resulting tax is $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly derived $61,701 of taxable income but used 4.25% rather than 3.99%, then abandoned even that $2,622.29 result and submitted an unsupported $3,846. The required calculation yields $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $30,000 joint standard deduction instead of $25,500 and rounded the rate to 4.0% instead of applying 3.99%. North Carolina taxable income is $61,701, producing $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $79,212 by subtracting the spouse's separately listed $7,989 employer-sponsored insurance premiums from the stated $46,000 of gross wages. PolicyEngine's AGI remains $87,201, so the $25,500 deduction leaves $61,701 taxed at 3.99%." -us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $3,717 is inconsistent with applying the North Carolina rate to $87,201 of AGI after the $25,500 joint deduction. The correct base is $61,701 and the correct 3.99% rate produces $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $79,212 as AGI, effectively removing the $7,989 employer-sponsored premium from wages that were already supplied as the annual wage input. AGI is $87,201, leaving $61,701 after the joint deduction and $2,461.87 of tax." -us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly set federal AGI to $79,212 by deducting the separately reported $7,989 employer-sponsored premium from the given wages. Using $87,201 of AGI and the $25,500 joint deduction gives $61,701 of taxable income and $2,461.87 of tax." -us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used $79,212 rather than $87,201 as North Carolina AGI, thereby understating taxable income by $7,989. The stated wages and $1 of self-employment income produce $61,701 after the $25,500 deduction, taxed to $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model's unexplained $2,551 estimate does not apply the specified North Carolina parameters. AGI of $87,201 less the $25,500 joint deduction equals $61,701, and 3.99% of that amount is $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model incorrectly concluded that the absence of itemized deductions or credits eliminates North Carolina tax liability. The household instead receives the $25,500 standard deduction and owes 3.99% of the remaining $61,701, or $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model understated state income at $79,212 by subtracting the $7,989 employer-sponsored insurance premium from the supplied wage amount. The correct AGI is $87,201, so taxable income is $61,701 and tax is $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model improperly applied a pre-tax insurance exclusion to the stated annual wages, reducing the tax base by $7,989. With $87,201 of AGI and the $25,500 joint deduction, North Carolina taxes $61,701 at 3.99% for $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model's rounded $2,800 estimate does not use the applicable 3.99% rate and $25,500 joint deduction. Those parameters applied to $87,201 of AGI yield $61,701 of taxable income and $2,461.87 of tax." -us,scenario_102,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a projected $26,400 joint standard deduction instead of $25,500 and rounded the tax rate to 4% instead of 3.99%. The proper calculation is ($87,201 − $25,500) × 3.99% = $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model both understated AGI at $79,212 and failed to apply North Carolina's $25,500 joint standard deduction. The correct taxable income is $61,701, not $79,212, and the 3.99% tax is $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required structured-output contract. -us,scenario_102,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used a $23,400 joint standard deduction instead of $25,500 and a 4.25% rate instead of 3.99%. The correct taxable income is $61,701 and the resulting liability is $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a 4.5% rate instead of 3.99% and a $23,850 joint standard deduction instead of $25,500. It also treated employer premiums as potential itemized medical expenses even though the applicable state standard deduction directly leaves $61,701 of taxable income, producing $2,461.87." +us,scenario_102,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly derived $5,406.40 of Social Security tax and $1,264.40 of Medicare tax, whose exact sum is $6,670.80, but replaced that result with an unrelated $6,539.00 submission. The claimed rounding cannot transform $6,670.80 into $6,539.00." +us,scenario_102,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model truncated total wages from $87,200 to $87,000 when calculating both Social Security and Medicare taxes, then submitted $6,681.00 even though its stated rounded components sum to $6,656.00. Using the exact wage inputs produces $6,670.80." +us,scenario_102,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented a $31 minimum or rounding treatment after acknowledging that $1 of self-employment income produces approximately zero tax. Net earnings at this level do not reach the self-employment-tax filing threshold, so the liability is $0." +us,scenario_102,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the requested self_employment_tax output entirely. The listed $1 of self-employment income is below the threshold for self-employment tax, yielding $0." +us,scenario_102,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model invented a PolicyEngine rule granting Medicare eligibility to every disabled person regardless of age. At age 26, the spouse does not satisfy the age threshold, and the listed disability alone does not establish Medicare eligibility." +us,scenario_102,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated disability as an automatic substitute for the age-65 Medicare criterion. The spouse is 26, and no qualifying under-65 Medicare entitlement is specified, so the eligibility value is 0." +us,scenario_102,spouse_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,The model incorrectly converted the spouse's disability flag directly into Medicare eligibility despite the spouse being under 65. Disability alone is not the applicable eligibility pathway in this computation. +us,scenario_102,spouse_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model applied an overbroad age-or-disability rule and concluded that any disabled person qualifies for Medicare. The spouse is only 26, and the disability fact does not independently satisfy the Medicare eligibility computation." +us,scenario_102,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly derived $61,701 of taxable income and identified the 3.99% rate, which yields $2,461.87, but discarded that calculation for an unexplained $2,755. Its alternative-base and 4.25% discussion does not produce its submitted amount." +us,scenario_102,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model falsely treated North Carolina as having no individual income tax in 2026. North Carolina taxes the household's $61,701 of taxable income at 3.99%, producing $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used 4.25% instead of North Carolina's 2026 rate of 3.99%. It also submitted $2,587 even though its stated $61,701 base at 4.25% equals about $2,622, so the final number follows neither the applicable rate nor its own arithmetic." +us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model explicitly computed the correct $61,701 taxable income and 3.99% rate, yielding about $2,462, then submitted $3,389. The submitted value is disconnected from every calculation in its explanation." +us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly stated the $61,700 taxable base and calculated tax of about $2,462 at 3.99%, then replaced it with $2,792 without a computation. Its final value contradicts its own correct derivation." +us,scenario_102,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used an obsolete $21,500 joint standard deduction instead of $25,500, overstating taxable income by $4,000. It compounded that error by applying 4.25% rather than the 2026 rate of 3.99%." +us,scenario_102,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly derived $61,701 of taxable income, but first applied the wrong 4.25% rate and then submitted $3,846 without any supporting calculation. The applicable 3.99% rate produces $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $30,000 joint standard deduction instead of North Carolina's $25,500 deduction, understating taxable income by $4,500. It also rounded the applicable 3.99% rate to 4%, yielding the wrong liability." +us,scenario_102,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $79,212 by subtracting the spouse's $7,989 employer-sponsored insurance premiums from the separately stated $46,000 of wages. The prompt's wage amounts enter AGI as given, so AGI is $87,201 and taxable income after the $25,500 deduction is $61,701." +us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $3,717 does not result from applying North Carolina's 2026 parameters to the household facts. The correct base is $61,701 and the correct rate is 3.99%, producing $2,461.87; the answer implies an overstated base or rate." +us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model reduced AGI from $87,201 to $79,212 by treating the separately listed $7,989 employer-sponsored insurance premium as an additional subtraction from stated wages. Using the stated wages and $1 of self-employment income leaves $61,701 taxable after the $25,500 joint deduction." +us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $7,989 employer-sponsored insurance premium from the stated wage amounts, producing AGI of $79,212 instead of $87,201. The $25,500 joint deduction therefore leaves $61,701, not $53,712, subject to the 3.99% rate." +us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used $79,212 of AGI, exactly $7,989 below the $87,201 generated by the listed wages and self-employment income. It double-counted the employer-sponsored premium as a deduction from wages, causing taxable income to be understated by $7,989." +us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model supplied only an unsupported estimate of $2,551 instead of applying the specified state parameters. North Carolina's $25,500 joint deduction leaves $61,701 taxable, and 3.99% of that amount is $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that the absence of listed itemized deductions or credits eliminates North Carolina income tax. The household instead claims the $25,500 state standard deduction and owes 3.99% on the remaining $61,701." +us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model understated state income by $7,989, using $79,212 rather than the $87,201 from stated wages and self-employment income. It treated the separately listed employer-sponsored premium as a further reduction of wages, leaving an erroneous $53,712 taxable base." +us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model subtracted the $7,989 employer-sponsored insurance premium from the wage totals even though the stated wages enter AGI as given. That double subtraction reduced the taxable base from $61,701 to $53,712 and produced $2,143.10 instead of $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave a coarse $2,800 estimate without identifying the applicable deduction or rate. Applying the $25,500 joint deduction to $87,201 of AGI and taxing $61,701 at 3.99% yields $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a projected $26,400 joint standard deduction instead of North Carolina's $25,500 amount, understating taxable income by $900. It also used a rounded 4% rate instead of 3.99%." +us,scenario_102,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model both reduced AGI to $79,212 by subtracting the $7,989 insurance premium and failed to apply North Carolina's $25,500 joint standard deduction. North Carolina taxes $61,701 at 3.99%, not the undeducted $79,212 at 4%." +us,scenario_102,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so its response failed the required output contract." +us,scenario_102,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used a $23,400 joint standard deduction instead of $25,500, overstating taxable income by $2,100. It also applied 4.25% rather than North Carolina's 2026 rate of 3.99%." +us,scenario_102,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model applied an obsolete 4.5% rate instead of 3.99% and used a $23,850 joint deduction instead of $25,500. Its medical-itemization discussion also incorrectly counted employer-sponsored premiums among medical deductions, though it ultimately did not use that itemization." +us,scenario_102,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model substituted the $30,000 federal standard deduction for North Carolina's $25,500 joint standard deduction. It then used an invented 4.49% state rate instead of the applicable 3.99%, and its submitted $2,567 does not exactly match even its stated arithmetic." us,scenario_102,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_104,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented a New York medically needy Medicaid pathway in PolicyEngine and treated it as making the aged head eligible despite the engine assigning medicaid_category = NONE. It also treated aged status plus spend-down logic as sufficient, while PolicyEngine found no applicable Medicaid category for this New York household." -us,scenario_104,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded veterans benefits from aged Medicaid countable income and concluded the head's Social Security alone fell below an aged/disabled income limit. PolicyEngine did not place the head in any aged Medicaid eligibility category, so this VA-benefit exclusion shortcut produced eligibility where the engine computed medicaid_category = NONE." -us,scenario_104,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly calculated net income of about $1,956 per month and explicitly recognized that this exceeds the roughly $1,255 net-income limit, but then awarded $2,052 anyway. A household that fails the net-income test is ineligible and cannot receive the minimum allotment." -us,scenario_104,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model understated countable net income by using an unsupported medical deduction of roughly $340 per month and then treated the remaining income as producing a $196 monthly allotment. Applying the allowable deductions to both Social Security and veterans benefits still leaves income above the one-person net-income limit, producing no SNAP eligibility or benefit." -us,scenario_104,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly treated the household’s relatively low monthly rent as producing no real property tax credit. Applying New York’s senior-renter real property tax credit formula to $6,483.53 of annual rent yields the $375 refundable credit." -us,scenario_104,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model limited New York refundable credits to earned-income, dependent-care, dependent, and NYC credits. It omitted the state real property tax credit available to this 72-year-old renter, which equals $375." -us,scenario_104,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model incorrectly counted the $20,520 of veterans benefits in the income amount it used to deny the New York real property tax credit. It then introduced an unsupported $63 estimate even though the applicable senior-renter calculation yields a $375 refundable credit." -us,scenario_104,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model treated zero New York taxable income as eliminating the real property tax credit. That refundable renter credit is determined separately from taxable income, and the head’s age and $6,483.53 rent produce $375." -us,scenario_104,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model incorrectly made earnings, positive AGI, or qualifying children necessary for every New York refundable credit. The separate real property tax credit applies to this elderly renter and yields $375." -us,scenario_104,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model denied the real property tax credit by adding veterans benefits to Social Security and comparing $34,616 with the $18,000 household-gross-income limit. That treatment wrongly removes the senior-renter pathway that produces a $375 credit from the household’s age and rent." -us,scenario_104,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,The model conflated the absence of taxable or earned income with the absence of refundable credits and dismissed the real property tax credit without applying its renter formula. The age-72 renter receives $375 under that formula. -us,scenario_104,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model stopped after finding no earned income for the New York EITC. It omitted the independent real property tax credit for the elderly renter, which contributes $375." -us,scenario_104,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,The model treated earned income and qualifying dependents as necessary for all state refundable credits. New York’s real property tax credit instead uses the elderly renter’s housing and household characteristics and yields $375. -us,scenario_104,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The zero answer omits New York’s real property tax credit pathway. Applying it to the 72-year-old renter and $6,483.53 of annual rent yields $375." -us,scenario_104,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model incorrectly treated the household as exceeding the real property tax credit’s $18,000 income limit, reflecting inclusion of veterans benefits in the disqualifying amount. The applicable senior-renter credit calculation yields $375." -us,scenario_104,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,The model’s blanket finding that no state refundable credit applies omits New York’s real property tax credit. The head’s age and annual rent generate a $375 refundable amount. -us,scenario_104,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,The model incorrectly concluded that household gross income exceeded the limit governing the New York real property tax credit. Applying the eligible senior-renter pathway produces $375. -us,scenario_104,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model invented a minimum-rent requirement and denied the real property tax credit because annual rent was supposedly too low. New York’s formula applied to $6,483.53 of rent for this 72-year-old renter yields $375." -us,scenario_104,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,The model failed to identify the New York real property tax credit indicated by the head’s age and rent. That credit supplies the full $375 of state refundable credits. -us,scenario_104,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,The model focused on earned-income-related credits and omitted the independent New York real property tax credit. The senior-renter formula produces $375. -us,scenario_104,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model incorrectly denied the real property tax credit because average monthly rent supposedly exceeded a renter limit. The applicable New York calculation uses this renter’s $6,483.53 annual rent and age 72 to produce $375." -us,scenario_104,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The provided age and rent establish the relevant New York real property tax credit pathway. The model overlooked that pathway, whose calculation yields $375." -us,scenario_104,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model wrongly treated earnings or children as the only bases for a refundable New York credit. This elderly renter qualifies through the real property tax credit calculation, producing $375." -us,scenario_104,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,The model incorrectly inferred that an elderly household without earnings or children has no refundable-credit eligibility. New York’s real property tax credit specifically supplies a renter pathway that yields $375 here. -us,scenario_104,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model limited refundable credits to those requiring qualifying income or children. It omitted the senior-renter real property tax credit, which equals $375." -us,scenario_104,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model denied the real property tax credit by treating household gross income as exceeding $18,000, reflecting inclusion of veterans benefits in the disqualifying amount. The applicable New York senior-renter calculation produces $375." -us,scenario_104,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,The model considered child-related eligibility but failed to apply New York’s real property tax credit for renters. The 72-year-old head’s rent produces a $375 refundable credit. -us,scenario_104,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model correctly ruled out earnings, child, tuition, and care credits but incorrectly concluded that the real property tax credit’s income limit was exceeded. Applying the eligible senior-renter pathway yields $375." -us,scenario_104,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model used $34,616 as household gross income by adding veterans benefits to Social Security and therefore incorrectly denied the renter real property tax credit under the $18,000 limit. The proper senior-renter calculation yields $375." -us,scenario_104,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model treated the absence of earnings and children as eliminating every state refundable credit. It omitted New York’s real property tax credit for this elderly renter, which yields $375." -us,scenario_104,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly asserted that the New York real property tax credit requires earned income or dependent children. This separate renter credit applies based on the household’s age, rent, and other eligibility inputs and equals $375." -us,scenario_107,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model overstated provisional income and taxable Social Security by incorporating the disability benefits into its combined-income calculation, yielding $3,335.50 instead of the taxable Social Security amount embedded in the $15,439.75 AGI. It also omitted the 2026 senior deduction, so it incorrectly left $1,577.50 of taxable income." -us,scenario_107,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security by using combined income that included part of the separately listed disability benefits, raising AGI to about $17,178 instead of $15,439.75. It then omitted the 2026 senior deduction and incorrectly produced $1,728 of taxable income." -us,scenario_107,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so it failed the required submission contract rather than completing the tax calculation." -us,scenario_107,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model stated that it applied the standard deduction, the age-65 additional standard deduction, and the temporary senior deduction but still left taxable income. Those deductions exceed the $15,439.75 AGI, so their correct application reduces taxable income and regular income tax to zero." -us,scenario_107,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the full $6,948 of disability benefits as taxable gross income and also used that amount in computing taxable Social Security, inflating AGI to $26,263 instead of $15,439.75. It further omitted the 2026 senior deduction, leaving a spurious $8,888 of taxable income." -us,scenario_107,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model did not submit a parseable value for federal_refundable_credits, so it failed the output contract rather than applying the refundable-credit rules." -us,scenario_107,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model computed a residual childless EITC from wages and an understated AGI of $15,440.50, excluding the household's full income from the phase-out calculation. With approximately $49,500 of household income for a single filer with no qualifying children, the EITC is fully phased out, and there are no dependent-child or education facts supporting any other refundable credit." -us,scenario_107,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented an Ohio age-65-plus Medicaid category and treated $2,000 in assets plus total income as within an elderly eligibility threshold. PolicyEngine places the person in medicaid_category = NONE, and their 2.67 FPL income does not qualify them through any Ohio Medicaid pathway." -us,scenario_107,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated being age 75 as automatically qualifying the person for an aged Medicaid pathway and then applied an unspecified higher FPL-based senior limit. PolicyEngine does not assign any aged, disabled, SSI, or MAGI Medicaid category, so the person has no categorical pathway despite being 75." -us,scenario_107,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly merged aged/disabled Medicaid with Medicare Savings Program-style reasoning and assumed the person's income was near SSI-linked limits after minimal disregards. The trace shows SSI received is 0, medicaid_category = NONE, and the person's income level is 2.67 FPL, so no Ohio aged/disabled Medicaid pathway is satisfied." -us,scenario_107,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model reduced the Medicaid decision to age, low earned income, and limited assets, ignoring unearned income and the need for a qualifying Ohio Medicaid category. PolicyEngine counts the case as 2.67 FPL for Medicaid purposes and assigns no eligibility category, yielding no Medicaid eligibility." -us,scenario_107,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable head_medicaid_eligible value or supporting explanation. This is a missing-output contract failure rather than a substantive Medicaid eligibility calculation. +us,scenario_104,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented automatic eligibility through New York’s medically needy spend-down pathway after its own income calculation exceeded the aged/disabled limit. PolicyEngine assigns no Medicaid category, so neither age nor the existence of a spend-down program converts this household into an eligible Medicaid case." +us,scenario_104,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age 72 plus Social Security below an approximate aged/disabled threshold as sufficient and categorically removed all $20,520 of veterans benefits from its income test. The applicable category determination yields NONE, with no SSI-based or other aged pathway establishing eligibility." +us,scenario_104,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly calculated net income of about $1,956 per month and explicitly found that it exceeded the elderly household's net-income limit, but then awarded $2,052 instead of enforcing the failed eligibility test. The minimum allotment applies only to an eligible one- or two-person household; it cannot replace a failed net-income test." +us,scenario_104,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model did not carry the listed $8,389 annual employer-sponsored insurance premium through the elderly medical-expense deduction and then asserted an unsupported $196 monthly benefit. With all $34,616 of Social Security and veterans income counted, the allowable deductions still leave net income above the one-person limit, while the $540 monthly rent is below half of adjusted income and creates no excess-shelter deduction." +us,scenario_104,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model wrongly treated low monthly rent as producing no real property tax credit. The senior renter formula applied to $6,483.53 of annual rent yields the $375 refundable credit." +us,scenario_104,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model considered earned-income and dependent-related credits but omitted the refundable real property tax credit. A 72-year-old qualifying renter can receive that credit without earnings or dependents, and this household receives $375." +us,scenario_104,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $20,520 of veterans benefits in household gross income and therefore denied the real property tax credit under the $18,000 limit. It then inserted an unsupported $63 despite its own conclusion that no refundable credit applied; the qualifying senior renter calculation yields $375." +us,scenario_104,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,The model incorrectly made positive New York taxable income or earned income a prerequisite for refundable credits. The real property tax credit is available to this qualifying senior renter and equals $375. +us,scenario_104,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated zero AGI, no earnings, and no children as eliminating every refundable credit. It omitted the renter-based real property tax credit, whose senior formula produces $375." +us,scenario_104,state_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the listed veterans benefits in household gross income, raising its total from qualifying income below $18,000 to $34,616. Excluding those benefits leaves the 72-year-old renter eligible for the $375 real property tax credit." +us,scenario_104,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,The model focused on EITC eligibility and incorrectly dismissed the real property tax credit based on nontaxable income and filing thresholds. The senior renter credit does not require earnings or positive taxable income and equals $375 here. +us,scenario_104,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model equated the absence of earned income with the absence of all refundable New York credits. It omitted the real property tax credit for the 72-year-old renter, which equals $375." +us,scenario_104,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model wrongly required earned income or qualifying dependents for every state refundable credit. New York's real property tax credit instead uses the senior renter's qualifying income and rent, producing $375." +us,scenario_104,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer omitted the New York real property tax credit eligibility pathway. The head's age, qualifying income, and $6,483.53 rent generate a $375 refundable credit." +us,scenario_104,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model exceeded the $18,000 real property tax credit limit only by counting the veterans benefits in household gross income. Those benefits are not counted in this calculation, so the senior renter qualifies for $375." +us,scenario_104,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,The zero answer omitted the refundable real property tax credit. Applying its senior renter eligibility and rent calculation yields $375. +us,scenario_104,state_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that household gross income exceeded the real property tax credit limit. The applicable calculation excludes the veterans benefits, leaving qualifying income below the limit and producing a $375 credit." +us,scenario_104,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model invented a minimum-rent threshold that disqualified the household. The $6,483.53 annual rent is a qualifying basis under the senior renter formula and produces the $375 real property tax credit." +us,scenario_104,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model overlooked the listed age and rent facts that establish the real property tax credit pathway. Applying that pathway gives a $375 refundable state credit. +us,scenario_104,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model restricted its review to earned-income-related and unspecified credits and omitted the renter circuit-breaker credit. The 72-year-old renter receives $375 under the real property tax credit formula. +us,scenario_104,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model wrongly asserted that average monthly rent above a renter limit bars the real property tax credit. The applicable senior renter formula uses the household's qualifying annual rent and yields $375. +us,scenario_104,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model failed to use the provided age and rent to test New York's real property tax credit. Those facts establish a $375 refundable credit. +us,scenario_104,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,The model wrongly treated earnings and children as the only possible bases for a refundable New York credit. The senior renter real property tax credit requires neither and equals $375. +us,scenario_104,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model incorrectly concluded that an elderly household without earnings or children has no refundable-credit pathway. The head qualifies as a senior renter for the $375 real property tax credit. +us,scenario_104,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model wrongly required qualifying earnings or children for all New York refundable credits. The renter's age, qualifying income, and rent generate a $375 real property tax credit." +us,scenario_104,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted veterans benefits toward the $18,000 household-gross-income limit for the real property tax credit. Excluding them leaves the senior renter eligible for the $375 credit." +us,scenario_104,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model focused on the child credit and failed to apply the separate real property tax credit. This childless 72-year-old renter qualifies for $375 based on age, qualifying income, and rent." +us,scenario_104,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model considered only EITC and the Empire State Child Credit, incorrectly treating earnings and dependents as necessary for every refund. It omitted the senior renter real property tax credit of $375." +us,scenario_104,state_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model denied the real property tax credit by using an income total that counted the listed veterans benefits. Those benefits are excluded from the applicable household-gross-income calculation, and the senior renter receives $375." +us,scenario_104,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model formed $34,616 of household gross income by adding veterans benefits to Social Security and therefore failed the $18,000 limit. The real property tax credit calculation excludes the veterans benefits, leaving qualifying income below the limit and yielding $375." +us,scenario_104,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,The model wrongly treated earned income and children as prerequisites for any refundable state credit. It omitted the $375 real property tax credit available to the qualifying senior renter. +us,scenario_104,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model incorrectly grouped the real property tax credit with credits requiring earned income or dependent children. The real property tax credit instead applies through the senior renter pathway and equals $375. +us,scenario_104,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly treated tax liability, earnings, dependents, or a separate rent qualification as necessary for every refundable credit. The listed age and rent support the senior renter real property tax credit, which equals $375." +us,scenario_107,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model included half of the $6,948 disability benefit in combined income when calculating taxable Social Security, producing $3,335.50 instead of $1,597.75. It also substituted a personal exemption for the applicable senior deduction, so it created taxable income even though the available deductions exceed the correct $15,439.75 AGI." +us,scenario_107,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model included half of the disability benefit in provisional income, overstating taxable Social Security as about $3,336 and AGI as about $17,178. It then understated the applicable deductions by omitting the senior deduction, whereas total deductions exceed the correct $15,439.75 AGI." +us,scenario_107,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no numeric value or explanation for the requested output, violating the required output contract." +us,scenario_107,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model stated that it applied the standard deduction, age-65 addition, and temporary senior deduction but failed to recognize that those deductions exceed the $15,439.75 AGI. Their application leaves zero taxable income, not an amount generating $171.25 of regular tax." +us,scenario_107,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $6,948 disability benefit as taxable gross income and also used it to inflate taxable Social Security to $5,473. It further omitted the applicable senior deduction and used a projected standard-deduction figure, creating $8,888 of taxable income instead of zero." +us,scenario_107,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $6,948 disability benefit in AGI and in the provisional-income calculation that produced about $5,473 of taxable Social Security. Excluding that benefit yields $1,597.75 of taxable Social Security and $15,439.75 of AGI, which is fully eliminated by the deductions the model itself listed." +us,scenario_107,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for federal_refundable_credits, violating the required output contract." +us,scenario_107,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model incorrectly carried the temporary 2021 removal of the childless EITC upper age limit into 2026, even though a 75-year-old is outside the applicable age range. It also understated AGI by omitting the taxable pension and applicable taxable Social Security income, placing the filer inside the phaseout range when the household’s income exceeds the childless EITC cutoff." +us,scenario_107,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated age 65+, $2,000 of assets, and $49,500 of annual income as sufficient for Ohio's aged Medicaid pathway without applying its actual income standard. The head qualifies through no Medicaid category, so satisfying the immigration rule and asserting that assets are below a limit cannot establish eligibility." +us,scenario_107,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model invented a higher Ohio aged/disabled income limit that covers the head's combined income and never identified or applied an actual qualifying threshold. The aged status creates a pathway to test, not automatic qualification, and the engine's category determination is NONE." +us,scenario_107,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated full Medicaid eligibility with Medicare Savings Program pathways such as QMB, SLMB, and QI, then treated the $20 exclusion and a $2,000 resource balance as decisive. Those pathways do not establish the requested Medicaid category here, and the head receives no SSI and qualifies through none of the Medicaid pathways." +us,scenario_107,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model inferred Medicaid eligibility from old age, low wages, and limited reported assets while ignoring the head's Social Security, pension, and disability income and the need to qualify through a defined category. The complete eligibility computation assigns category NONE and returns no eligibility." +us,scenario_107,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for head_medicaid_eligible, violating the required output contract." us,scenario_107,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_107,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model did not return a parseable payroll_tax value, so it failed the required submit_outputs contract rather than making a substantive payroll-tax computation." -us,scenario_107,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified the employee Social Security and Medicare rates and even computed a subtotal near $486.96, but then submitted $545 with a false rounding explanation. Its submitted value is inconsistent with its own payroll-tax arithmetic and adds about $58 beyond employee FICA with no applicable Additional Medicare Tax or state payroll tax pathway." -us,scenario_107,payroll_tax,minimax-m3,llm_error,thresholds_rates,False,"The model treated the required exact annual payroll tax as a rounded estimate, converting an approximately $487 employee FICA calculation into $490. PolicyEngine reports the exact employee Social Security plus Medicare result, $394.66 + $92.30 = $486.96, with no coarse rounding step." -us,scenario_107,snap,claude-opus-5,llm_error,thresholds_rates,False,The model acknowledged that monthly income far exceeded the income limit but then incorrectly used the uncapped elderly-household shelter deduction to create eligibility. It failed to apply the income-threshold denial and therefore calculated a $127 monthly allotment for an ineligible household. -us,scenario_107,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model characterized $49,500 of annual income as very low relative to need and rent, bypassing the applicable SNAP income threshold. Rent does not erase the threshold failure, so the positive $2,700 estimate came from treating housing costs as sufficient to establish eligibility." -us,scenario_107,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model improperly used the earned-income, standard, and uncapped excess-shelter deductions to reduce $4,125 of monthly gross income to $845 and bypass the SNAP income-threshold denial. It also misapplied the allotment formula: $292 minus 30% of $845 is about $38.50, not $292 per month." -us,scenario_107,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model applied a flat 2.75% Ohio tax rate to the first dollar of its computed $3,966 taxable income instead of applying Ohio's zero-tax bracket for low taxable income. That bracket threshold eliminates the entire Ohio liability before refundable credits, so the later $50 senior citizen credit calculation was applied to a liability that should never have existed." +us,scenario_107,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required submission contract." +us,scenario_107,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's own component calculation produced $486.84, but it then changed that total to $545 under the false claim that this was nearest-whole-dollar rounding. Employee Social Security and Medicare taxes sum to $486.96 in the engine trace, and rounding that amount cannot produce $545." +us,scenario_107,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly identified the 6.2% Social Security and 1.45% Medicare components but replaced their precise sum with an unauthorized ten-dollar-rounded estimate of $490. The required annual payroll tax is the engine's component sum of $394.66 and $92.30, or $486.96." +us,scenario_107,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model used the uncapped elderly-household shelter deduction to manufacture positive net-income eligibility after acknowledging that monthly income far exceeded the limit. The household fails the applicable income screen before that deduction can generate a benefit, yielding $0 rather than $1,524." +us,scenario_107,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly characterized approximately $49,500 of annual income as very low countable income and awarded SNAP based on rent without applying the one-person income eligibility screen. Gross monthly income of about $4,125 exceeds the threshold, so the benefit is $0." +us,scenario_107,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model improperly treated the earned-income, standard, shelter, and utility deductions as overcoming the household’s initial income ineligibility. It also contradicted its own formula: $292 minus 30% of $845 is about $38.50, not $292, so $3,504 is the maximum allotment rather than the result of its stated calculation." +us,scenario_107,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model applied Ohio's 2.75% rate from the first dollar of its calculated $3,966 taxable income. That amount falls entirely within Ohio's zero-tax bracket, so tax before the senior credit is already $0 and cannot become $59.07." us,scenario_107,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_108,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $10,392 `survivor benefits` input to taxable income as a non-Social-Security pension, raising AGI from $7,656 to $18,048. Its subsequent use of an estimated $17,000 deduction created $1,048 of taxable income that does not exist." -us,scenario_108,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model treated the $10,392 `survivor benefits` amount as taxable federal income and therefore calculated AGI as $18,048 instead of $7,656. That erroneous inclusion alone produced its $1,248 taxable-income base and $124.80 tax." -us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model misclassified the $10,392 `survivor benefits` input as a taxable survivor pension, inflating AGI to $18,048 rather than $7,656. It also applied an obsolete post-TCJA-sunset deduction structure, but no taxable income remains once the correct AGI is used." -us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $10,392 `survivor benefits` amount in AGI and then assumed a TCJA-expiration deduction regime. PolicyEngine's AGI is only $7,656, which is fully absorbed by the applicable deduction and yields no taxable income." -us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model treated the $10,392 `survivor benefits` input as taxable income, producing an erroneous $18,048 AGI. Its approximate $15,800 deduction calculation therefore created $2,248 of taxable income instead of zero." -us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly excluded Social Security after its provisional-income test but incorrectly included the separate $10,392 `survivor benefits` input as taxable income. That raised AGI from $7,656 to $18,048 and created the spurious $348 taxable-income remainder." -us,scenario_108,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model misclassified the $10,392 `survivor benefits` amount as a taxable survivor pension and calculated AGI of $18,048 instead of $7,656. Its estimated deduction then left a fictitious $598 taxable-income base." -us,scenario_108,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly added the $10,392 `survivor benefits` input to federal AGI and compounded that error by applying a projected post-TCJA-sunset standard deduction and personal exemption. At the correct $7,656 AGI, the applicable deduction eliminates taxable income." -us,scenario_108,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $10,392 `survivor benefits` amount as taxable federal income, inflating AGI to $18,048. Its speculative standard deduction, age addition, and personal exemption then produced $3,291 of taxable income, whereas the correct $7,656 AGI yields none." -us,scenario_108,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model excluded Social Security correctly but incorrectly included the separate $10,392 `survivor benefits` input in AGI. That error raised AGI to $18,048 and left the $348 taxable-income balance underlying its $35 answer; the correct AGI is $7,656 and taxable income is zero." -us,scenario_108,federal_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model computed a childless-worker EITC from wages using the 7.65% phase-in rate but skipped the EITC age eligibility rule for claimants with no qualifying children. An 85-year-old without qualifying children is outside the childless EITC age band, so the EITC component is $0 and no other refundable federal credit applies." -us,scenario_108,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not submit a parseable value or explanation for federal_refundable_credits. Under the required output contract, the missing answer prevents any substantive tax-rule calculation from being evaluated." +us,scenario_108,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model misclassified the $10,392 survivor-benefit input as taxable non-Social-Security pension income, inflating AGI from $7,656 to $18,048. With the correct AGI, the deduction fully eliminates taxable income, so its $105 liability does not arise." +us,scenario_108,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model added the $10,392 survivor benefits to taxable wages even though that input does not enter federal AGI. Correct AGI is $7,656, which is fully offset by the applicable deduction rather than leaving the $1,248 of taxable income it computed." +us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model treated the $10,392 survivor benefits as taxable pension income and consequently started from an erroneous $18,048 AGI instead of $7,656. Its post-TCJA-style deduction and personal-exemption calculation cannot create tax because the correctly classified income is already below the applicable deduction." +us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $10,392 survivor benefits in federal AGI as taxable income. Correct AGI contains only the $7,656 of wages, and the applicable deduction reduces taxable income to zero." +us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model misclassified the $10,392 survivor-benefit amount as taxable pension income, producing $18,048 of AGI. Excluding that nontaxable input leaves $7,656 of AGI and no taxable income after deductions." +us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Although the model correctly excluded Social Security, it wrongly treated the separate $10,392 survivor-benefit input as taxable income. The correct AGI is $7,656 rather than $18,048, so the deduction eliminates all taxable income instead of leaving $348." +us,scenario_108,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model erroneously labeled the $10,392 survivor benefits as a taxable survivor pension and included them in AGI. With AGI of $7,656 rather than $18,048, there is no $598 taxable-income remainder after the deduction." +us,scenario_108,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly included the $10,392 survivor benefits as taxable income, inflating AGI to $18,048. The correct $7,656 AGI is fully absorbed by the applicable deduction, regardless of its mistaken TCJA-sunset deduction framework." +us,scenario_108,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model added the $10,392 survivor benefits to federal AGI as though they were taxable pension payments. Correct AGI is only $7,656, so deductions eliminate taxable income and the model's $3,291 taxable-income calculation never arises." +us,scenario_108,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model correctly excluded Social Security but incorrectly included the separate $10,392 survivor benefits in AGI. Correct AGI is $7,656, which the applicable deduction reduces to zero taxable income rather than the $348 it calculated." +us,scenario_108,federal_refundable_credits,grok-build-0.1,llm_error,age_disability,False,"The model applied the 7.65% childless-EITC phase-in rate without enforcing the childless claimant age limit. At age 85, the head is ineligible for the childless EITC, so the computed $586 EITC must be $0." +us,scenario_108,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits. The required result is $0 because every refundable-credit component evaluates to zero, including EITC for this 85-year-old claimant with no qualifying children." us,scenario_108,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_108,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_108,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model inferred an aged/disabled pathway from age, disability, and low assets without establishing any Medicaid category. The engine assigns category NONE, so income exclusions and asset limits never yield eligibility." -us,scenario_108,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated being age 85 and disabled as sufficient entry into Wisconsin aged/disabled Medicaid. The head qualifies through no modeled category, so its assertion that income is below an applicable limit does not establish eligibility." -us,scenario_108,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model assumed age and disability automatically activated an ABD Medicaid pathway and then applied SSI-related disregards. The head receives no SSI and matches no Medicaid category, so no ABD income test produces eligibility." -us,scenario_108,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model invented eligibility through MAPP or a medically needy spend-down without establishing that either pathway applies under the modeled rules. The $800 of stated medical expenses, age, disability, surviving-spouse status, and IHS coverage do not change the engine's category NONE result." -us,scenario_108,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated age and disability as sufficient for Wisconsin SSI-related Medicaid and proceeded directly to financial limits. The head receives no SSI and matches no Medicaid category, so low income and $270 of assets do not establish eligibility." -us,scenario_108,head_medicaid_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model substituted Wisconsin MAPP's disabled-worker pathway for PolicyEngine's actual category determination. Despite wages and disability, the head matches no modeled Medicaid category, so the asserted 250%-of-FPL MAPP limit is not an operative eligibility test." -us,scenario_108,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model inferred Medicaid eligibility directly from advanced age, low income, and minimal assets. Those facts do not supply a modeled eligibility category; the head's category is NONE." -us,scenario_108,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated age, disability, and Indian Health Service coverage as establishing Medicaid eligibility. IHS coverage is not a Medicaid eligibility category, and the head qualifies through none of the modeled pathways." -us,scenario_108,head_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model applied earned-income exclusions under an elderly/disabled income test without first establishing an eligible category. The head matches no Medicaid category, so favorable income budgeting cannot make the head eligible." -us,scenario_108,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model treated disability, senior status, low income, and IHS coverage as collectively sufficient for Wisconsin Medicaid. None establishes a modeled category for this head, whose category is NONE." -us,scenario_108,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly placed the head in an SSI-related aged/blind/disabled category and treated approximately $23,326 as within its income limit. The head receives no SSI and qualifies through no Medicaid category; IHS coverage and age do not create that pathway." +us,scenario_108,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model inferred an aged/disabled Medicaid pathway from age, disability, and low assets without establishing a qualifying category. The head matches no Medicaid category, so income exclusions and assets cannot produce eligibility." +us,scenario_108,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated being age 85 and disabled as sufficient categorical qualification for Wisconsin aged/disabled Medicaid. The head matches no Medicaid eligibility category, regardless of its characterization of the income as low." +us,scenario_108,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly assigned the head to an aged/blind/disabled pathway and then applied SSI-related disregards. Age and disability do not create an engine-recognized category here, and the head receives no SSI." +us,scenario_108,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model invented qualification through an aged/disabled or medically needy spend-down pathway, even though the head matches no Medicaid category. The listed $800 of medical expenses, surviving-spouse status, and Indian Health Service coverage do not establish such a pathway." +us,scenario_108,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model assumed that age, disability, low assets, and asserted income exclusions place the head in Wisconsin’s SSI-related Medicaid pathway. The head receives no SSI and matches no Medicaid category, so that pathway does not apply." +us,scenario_108,head_medicaid_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model assigned the head to Wisconsin’s Medicaid Purchase Plan solely from disability, work, income, and assets. The head matches no Medicaid category under the applicable PolicyEngine pathways, so the asserted MAPP route cannot support eligibility." +us,scenario_108,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used elderly status, low income, and minimal assets as a shortcut to eligibility without identifying a qualifying Medicaid pathway. The head matches no Medicaid category, which ends the eligibility determination." +us,scenario_108,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated age, disability, and Indian Health Service coverage as establishing Medicaid eligibility. None supplies a qualifying Medicaid category for this head, and IHS coverage is not an eligibility pathway." +us,scenario_108,head_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model applied earned-income exclusions under an elderly/disabled budgeting pathway without first establishing categorical eligibility. The head matches no Medicaid category, so favorable income budgeting cannot make the head eligible." +us,scenario_108,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model inferred Medicaid eligibility directly from disability, senior status, low income, and IHS coverage. Those facts do not place the head in any Medicaid category, and IHS coverage does not create one." +us,scenario_108,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly placed the head in an SSI-related aged/blind/disabled category based on age, disability, IHS coverage, and its income estimate. The head receives no SSI and matches no Medicaid category; it also omitted the $7,656 of wages when describing countable income." us,scenario_108,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_108,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied the correct taxable wage base and correctly computed Social Security tax as $474.67, but it miscomputed Medicare tax: 1.45% of $7,656 is $111.01, not $110.01. That one-dollar Medicare arithmetic error drove its total to $584.68 instead of $585.68." -us,scenario_108,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model identified the right payroll-tax components but abandoned its own calculation of about $586 and submitted an unsupported rounded estimate of $605. Applying 6.2% Social Security and 1.45% Medicare to $7,656 yields $474.67 and $111.01, with no Additional Medicare Tax or Wisconsin employee payroll tax, totaling $585.68." -us,scenario_108,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a payroll_tax value or explanation. It therefore failed the required output contract rather than making a substantive payroll-tax computation. +us,scenario_108,payroll_tax,claude-sonnet-4.6,llm_error,other,False,"The model transcribed the Medicare calculation as $110.01 even though its own displayed multiplication, 1.45% × $7,656, equals $111.01. Adding $474.67 and $111.01 yields $585.68, exactly $1 above its answer." +us,scenario_108,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model identified approximately $475 of Social Security tax and $111 of Medicare tax, which sum to about $586, but then submitted $605 with no additional payroll-tax component. Using the exact components gives $474.67 + $111.01 = $585.68." +us,scenario_108,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required structured-output contract." us,scenario_108,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_108,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_108,snap,claude-fable-5,llm_error,categorical_eligibility,False,The model applied the ordinary 100% FPL net-income test as a disqualifier and omitted Wisconsin categorical eligibility through TANF non-cash assistance. That pathway preserves eligibility and triggers the minimum allotment despite the negative formula benefit. -us,scenario_108,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model disqualified the household under a 130% FPL income test and never applied categorical eligibility through TANF non-cash assistance. The eligible one-person household receives the monthly minimum allotment. -us,scenario_108,snap,claude-opus-4.7,llm_error,other,False,"The model correctly recognized that the negative formula amount leads to a minimum benefit, then abandoned that calculation and submitted $2,227 without a supporting derivation. The minimum monthly allotments sum to $287.68, not an average benefit near $186 per month." -us,scenario_108,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model explicitly found that the ordinary net-income test failed but omitted categorical eligibility through TANF non-cash assistance. It then submitted $2,376 even though its own reasoning declared the household ineligible, compounding the eligibility error with an unsupported output." -us,scenario_108,snap,claude-opus-5,llm_error,period_annualization,False,"The model identified the minimum-allotment outcome but annualized a fixed $23 monthly amount as $276. PolicyEngine aggregates benefit months with minimums of $23.84 and, after the guideline update, $24.37, yielding $287.68." -us,scenario_108,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model incorrectly asserted that Wisconsin categorical eligibility does not overcome the income tests and therefore treated the 100% FPL net limit as dispositive. TANF non-cash categorical eligibility keeps this household eligible for the minimum allotment. -us,scenario_108,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model claimed deductions reduced net income enough to generate a $164 monthly formula benefit, but its listed income and deductions leave net income around $2,036, making 30% of net income exceed the $298 maximum allotment. The resulting negative formula amount is replaced by the minimum allotment, not $164." -us,scenario_108,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model converted a negative maximum-allotment formula directly to zero. For this categorically eligible one-person household, the negative formula result is replaced by the minimum SNAP allotment." -us,scenario_108,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model treated high gross income and a negative formula allotment as producing no benefit. It omitted TANF non-cash categorical eligibility and the minimum allotment owed to an eligible one-person household. -us,scenario_108,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The answer implies the model used an ordinary income threshold as an absolute bar. Wisconsin categorical eligibility through TANF non-cash assistance makes the household eligible, after which the minimum allotment applies." -us,scenario_108,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model treated the gross and net income limits as dispositive and omitted categorical eligibility. The household remains eligible through TANF non-cash assistance and receives the one-person minimum allotment. -us,scenario_108,snap,gemini-3.5-flash,llm_error,period_annualization,False,"The model correctly selected the minimum-benefit pathway but used a flat $23 for all twelve months. The applicable monthly minimum changes within the year from $23.84 to $24.37, so the annual aggregation is $287.68." -us,scenario_108,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model treated income exceeding ordinary limits and a zero formula amount as eliminating SNAP. Categorical eligibility preserves entitlement, and the negative formula amount is floored at the minimum allotment." -us,scenario_108,snap,glm-5.2,llm_error,categorical_eligibility,False,The model correctly calculated that 30% of net income exceeds the maximum allotment but incorrectly floored the benefit at zero. An eligible one-person household qualifying categorically through TANF non-cash assistance receives the statutory minimum allotment. -us,scenario_108,snap,gpt-5.4-mini,llm_error,other,False,"The model supplied $1,172 without calculating countable net income, the expected contribution, or the minimum-benefit floor. The trace produces a negative regular allotment and therefore only the month-specific minimums totaling $287.68." -us,scenario_108,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model treated the absence of additional deductions as grounds for zero and omitted categorical eligibility through TANF non-cash assistance. Eligibility plus a negative regular allotment produces the minimum benefit. -us,scenario_108,snap,gpt-5.5,llm_error,categorical_eligibility,False,The model correctly concluded that the ordinary allotment formula is negative but incorrectly equated that result with zero SNAP. The categorically eligible one-person household receives the minimum allotment instead. -us,scenario_108,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $103 monthly estimate overstates the shelter and medical deductions or understates countable income. Net income is $2,035.67 monthly, so the expected contribution exceeds the $298 maximum and only the minimum allotment remains." -us,scenario_108,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model used failure of the ordinary elderly-or-disabled net-income limit to deny SNAP. TANF non-cash categorical eligibility bypasses that disqualification, leaving the household entitled to the minimum allotment." -us,scenario_108,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model treated the one-person net-income limit as an absolute eligibility condition. Wisconsin categorical eligibility through TANF non-cash assistance preserves eligibility and produces the minimum allotment. -us,scenario_108,snap,grok-4.3,llm_error,categorical_eligibility,False,The model denied SNAP solely because income exceeded the ordinary limit. It omitted categorical eligibility and the minimum-allotment rule for an eligible one-person household. -us,scenario_108,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model treated net income above 100% FPL as disqualifying. Categorical eligibility through TANF non-cash assistance makes that comparison non-dispositive, and the household receives the minimum allotment." -us,scenario_108,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model correctly computed that ordinary net income exceeds the 100% FPL limit but stopped there. TANF non-cash categorical eligibility preserves SNAP eligibility, so the negative regular allotment is floored at the minimum." -us,scenario_108,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model applied the ordinary 100% FPL net-income test as a final eligibility bar. The household qualifies categorically through TANF non-cash assistance and therefore receives the minimum allotment. -us,scenario_108,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the maximum allotment but incorrectly returned zero. Because categorical eligibility establishes entitlement, the one-person minimum allotment replaces the negative formula result." -us,scenario_108,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model denied eligibility under gross and net poverty tests and also omitted the 20% earned-income deduction from its stated net calculation. More fundamentally, TANF non-cash categorical eligibility makes those ordinary income-test conclusions non-dispositive and leads to the minimum allotment." -us,scenario_108,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model repeatedly recomputed deductions inconsistently, including inventing an unlisted utility allowance, but every version ended by converting a negative regular allotment to zero. The household's categorical eligibility requires application of the one-person minimum allotment, and only listed shelter expenses belong in the calculation." -us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model included all $10,392 of survivor-benefit income in Wisconsin taxable income and omitted the state’s retirement-income subtraction for an eligible taxpayer age 67 or older. After that subtraction, the remaining $7,656 of wages is fully absorbed by Wisconsin deductions and exemptions, yielding zero tax." -us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model started from $18,048 and applied only Wisconsin’s sliding-scale standard deduction, omitting the age-67-or-older retirement-income subtraction for the $10,392 of survivor benefits. That subtraction reduces income to $7,656 before the standard deduction and exemptions, leaving no taxable income." -us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model correctly excluded Social Security but still treated the separate $10,392 survivor benefit as Wisconsin taxable income, overlooking the retirement-income subtraction available at age 85. The renter-related school property tax credit is not the step that drives the result to zero; subtracting the survivor income before deductions eliminates taxable income." -us,scenario_108,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model applied the standard deduction and personal exemption to the full $18,048 but failed to subtract the $10,392 of qualifying retirement or survivor income under Wisconsin’s age-67-or-older retirement-income subtraction. With that income removed first, the remaining wages fall below the applicable deductions and exemptions." -us,scenario_108,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The answer implies that the model taxed the survivor-benefit income after only the standard deduction and personal exemption. Wisconsin’s retirement-income subtraction removes the $10,392 for this 85-year-old taxpayer, after which deductions eliminate the remaining wage income and the liability is zero." -us,scenario_108,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model expressly included the $10,392 survivor benefit in Wisconsin AGI and applied only a phased standard deduction. It omitted Wisconsin’s retirement-income subtraction for taxpayers age 67 or older, which removes that income and leaves no taxable income after the remaining deductions and exemptions." -us,scenario_108,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so it failed the required output contract." -us,scenario_108,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $10,392 survivor-benefits input in homestead income, inflating it from $20,590 to $30,982. It then abandoned its own zero-credit conclusion and substituted an unsupported $636 estimate instead of applying the formula to $1,440 of property tax and $20,590 of income." -us,scenario_108,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly excluded the Wisconsin Homestead Credit from refundable state income-tax credits because it provides property-tax relief. It is a refundable credit included in this output, and the eligible elderly and disabled renter receives $271.39." -us,scenario_108,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model excluded the $12,934 tax-exempt Social Security benefit from homestead income and instead counted the separate $10,392 survivor-benefits field, producing $18,048 rather than $20,590. Applying an approximate phaseout to that wrong income produced $300 instead of $271.39." -us,scenario_108,state_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model added both $12,934 of Social Security survivor benefits and the separate $10,392 survivor-benefits input to wages. Wisconsin homestead income is $20,590, not $30,982, so the income-limit denial was wrong." -us,scenario_108,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an approximate income of $18,000 and an unspecified approximate reduction formula. The calculation requires $20,590 of homestead income and $1,440 of rent-derived property tax, which yields $271.39 rather than $371." -us,scenario_108,state_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model double-counted the two survivor-benefit inputs, inflating homestead income to $30,982. The applicable income is $7,656 plus $12,934, or $20,590, which preserves homestead-credit eligibility." -us,scenario_108,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model overlooked the explicit Wisconsin Homestead Credit pathway for an elderly or disabled renter. The claimant's age, disability, $7,200 rent, and $20,590 homestead income generate a $271.39 refundable credit without any dependent requirement." -us,scenario_108,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reached $30,982 by adding the separate $10,392 survivor-benefits input to wages and Social Security. Homestead income is $20,590, so the claimant is below the limit and receives $271.39." -us,scenario_108,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated Wisconsin homestead income as $30,982 by double-counting survivor benefits. The correct $20,590 income does not disqualify the claimant and produces a $271.39 credit." -us,scenario_108,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer omitted the Wisconsin Homestead Credit eligibility pathway for an elderly, disabled renter. With $20,590 of homestead income and $1,440 of rent-derived property tax, that pathway yields $271.39." -us,scenario_108,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model double-counted the listed survivor-benefit amounts and treated homestead income as $30,982. The calculation uses $20,590, leaving the claimant eligible for $271.39." -us,scenario_108,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the separate $10,392 survivor-benefits input in addition to $12,934 of Social Security survivor benefits. This inflated homestead income from $20,590 to $30,982 and caused a false income-limit denial." -us,scenario_108,state_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used $30,982 as homestead income by adding both survivor-benefit entries to wages. The required income base is $20,590, which yields a positive $271.39 Homestead Credit." -us,scenario_108,state_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model double-counted survivor benefits and therefore applied the homestead income limit to $30,982. The correct income is $20,590; the absence of qualifying children only eliminates Wisconsin EITC and does not eliminate the $271.39 Homestead Credit." -us,scenario_108,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to apply the Wisconsin Homestead Credit triggered by the claimant's age, disability, rent, and qualifying income. Those facts produce $271.39 even though no other refundable Wisconsin credit applies." -us,scenario_108,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model restricted its analysis to credits resembling the EITC and omitted Wisconsin's refundable Homestead Credit. The elderly and disabled renter qualifies for $271.39 based on $1,440 of rent-derived property tax and $20,590 of homestead income." -us,scenario_108,state_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model double-counted the survivor-benefit fields and therefore concluded that income exceeded the homestead limit. Homestead income is $20,590, and qualifying children or childcare expenses are irrelevant to the claimant's $271.39 Homestead Credit." -us,scenario_108,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model treated the lack of qualifying children as eliminating every refundable Wisconsin credit. Wisconsin's Homestead Credit has an elderly or disabled claimant pathway, under which this renter receives $271.39." -us,scenario_108,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied the homestead income limit to an overstated income measure. The applicable $20,590 consists of wages plus tax-exempt Social Security benefits and supports a $271.39 credit." -us,scenario_108,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model overlooked that one-adult households can qualify for the Wisconsin Homestead Credit through age or disability. This 85-year-old disabled renter meets that pathway and receives $271.39. -us,scenario_108,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The blanket zero omitted the Wisconsin Homestead Credit. The claimant's elderly and disabled status, $7,200 rent, and $20,590 homestead income generate $271.39." -us,scenario_108,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted survivor benefits to derive approximately $30,982 of household income. The Homestead Credit uses $20,590 here, so the claimant remains eligible; the lack of federal EITC affects only Wisconsin EITC." -us,scenario_108,state_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model formed $30,982 by adding wages, Social Security survivor benefits, and the separate survivor-benefits input. The homestead calculation uses $20,590 and therefore yields $271.39 rather than an income-limit denial." -us,scenario_108,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric state_refundable_credits output or explanation. It therefore failed the required output contract rather than completing the Wisconsin Homestead Credit calculation. -us,scenario_108,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model double-counted the survivor-benefit entries to obtain approximately $30,982 of broad household income. The applicable homestead income is $20,590, and the absence of a childless Wisconsin EIC does not remove the $271.39 Homestead Credit." -us,scenario_108,state_refundable_credits,minimax-m3,llm_error,other,False,The model incorrectly made refundable-credit eligibility depend on having state income-tax liability. Wisconsin's Homestead Credit is refundable and produces $271.39 even when pre-credit state income tax is zero. -us,scenario_108,state_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model added the $10,392 survivor-benefits input on top of the $12,934 Social Security survivor benefit and wages, overstating homestead income as $30,982. The correct $20,590 income qualifies for a $271.39 Homestead Credit; the no-child EITC rule affects only the separate Wisconsin EITC." -us,scenario_109,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied only CHIP's under-19 rule and upper-income ceiling. It omitted the prerequisite that the child not qualify for Medicaid; this nine-year-old qualifies for Florida Medicaid in the OLDER_CHILD category, which makes CHIP unavailable." -us,scenario_109,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model computed MAGI below CHIP's upper limit but never tested Medicaid eligibility first. Child 1 qualifies for Florida Medicaid as an OLDER_CHILD, and that Medicaid eligibility excludes the child from CHIP." -us,scenario_109,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model recognized that CHIP covers income above the Medicaid limit but wrongly placed this household within that CHIP band. At approximately $30,000, Child 1 remains within Florida's Medicaid limit for the OLDER_CHILD category, so the child is not CHIP-eligible." -us,scenario_109,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model treated age under 19, income below 200% FPL, Florida residence, and no listed coverage as sufficient for CHIP. It omitted CHIP's Medicaid-ineligibility condition: Child 1 qualifies for Florida Medicaid as an OLDER_CHILD, which precludes CHIP." -us,scenario_109,child1_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model compared income only with CHIP's 200% FPL ceiling and ignored the lower boundary created by Medicaid eligibility. At this income, the nine-year-old qualifies for Florida Medicaid in the OLDER_CHILD category and therefore cannot qualify for CHIP." -us,scenario_109,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model failed to apply Florida's child Medicaid older child MAGI category to the 9-year-old dependent. It treated the absence of separately listed Medicaid indicators as disqualifying and ignored that the household's MAGI of 0.72 FPL is below the 133% FPL income threshold for older children. -us,scenario_109,child2_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model acknowledged that CHIP covers children who are not eligible for Medicaid but never applied that prerequisite. Child 2 qualifies for Medicaid under the OLDER_CHILD category, which makes CHIP eligibility false regardless of the model's CHIP-threshold comparison." -us,scenario_109,child2_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model tested only age and the CHIP maximum-income limit, treating income below roughly 200% FPL as sufficient. It omitted the required Medicaid-ineligibility test; Child 2 is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP." -us,scenario_109,child2_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model reduced CHIP eligibility to being under age 19 and below Florida's CHIP income ceiling. It failed to determine that Child 2 is Medicaid-eligible under the OLDER_CHILD category, which excludes the child from CHIP." -us,scenario_109,child2_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model combined the Medicaid and CHIP income ranges and concluded that falling within either range established CHIP eligibility. At this income, Child 2 qualifies for Medicaid under the OLDER_CHILD category, and that Medicaid eligibility makes the child ineligible for CHIP." -us,scenario_109,child2_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,The model applied only the under-19 requirement and Florida's CHIP upper-income threshold. It omitted CHIP's Medicaid-ineligibility prerequisite: Child 2 qualifies for Medicaid under the OLDER_CHILD category and is consequently excluded from CHIP. -us,scenario_109,child2_head_start_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an inflated 2026 poverty guideline for a family of five, stating about $36,580, and therefore classified $30,000 as below 100% FPL. PolicyEngine's Head Start income screen places this household above the applicable 100% poverty threshold, so child 2 fails income eligibility before any exception applies." -us,scenario_109,child2_head_start_eligible,claude-opus-4.8,llm_error,other,False,"The model's explanation says child 2 is not eligible because age 6 is outside Head Start's preschool age range, but it submitted value = 1. This is a contract failure: the submitted numeric answer contradicts its own eligibility explanation." -us,scenario_109,child2_head_start_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced self-employment income with the 92.35% self-employment tax adjustment for the Head Start income test, lowering countable income to about $27,705. PolicyEngine uses the household income level for the Head Start poverty comparison, and the household's about $30,000 income exceeds the applicable 100% FPL cutoff for eligibility." -us,scenario_109,child2_head_start_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model focused on a discretionary age-6 possibility and never applied the Head Start income screen. PolicyEngine denies eligibility because the household's about $30,000 income is above the applicable 100% federal poverty threshold and no categorical exception is listed." -us,scenario_109,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of explicitly listed Medicaid enrollment or special qualifying indicators as dispositive and never evaluated the child's categorical MAGI pathway. Child2 is a 6-year-old dependent in Florida's older-child Medicaid category, and household MAGI income is 0.72 FPL, below the 133% FPL limit." -us,scenario_109,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used the wrong Florida Medicaid income limit for a 6-year-old, applying a 100% FPL cutoff and then overriding even that with adult non-expansion and asset-test reasoning. PolicyEngine applies the older-child Medicaid category for child2 with a 133% FPL MAGI threshold, and this household's 0.72 FPL MAGI income satisfies it; the adult Medicaid expansion rule and the spouse's bank assets do not disqualify the dependent child from this MAGI category." -us,scenario_109,child2_wic_eligible,minimax-m3,llm_error,age_disability,False,"The model identified that Child 2 is age 6 and above the WIC child age limit, but then overrode that rule with an unsupported assumption that PolicyEngine counts the child as eligible. PolicyEngine applies the under-age-5 WIC child category, so a 6-year-old child is not WIC-eligible." -us,scenario_109,child3_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model treated infancy, lack of existing coverage, and CHIP's under-19 age rule as sufficient for eligibility. It omitted the required exclusion for children already eligible for Medicaid; child3 qualifies for Medicaid under the INFANT category and is therefore not CHIP-eligible." -us,scenario_109,child3_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model compared household income directly with a CHIP ceiling and labeled the resulting coverage range CHIP-funded. It failed to first apply the infant Medicaid pathway, under which child3 is Medicaid-eligible and consequently excluded from CHIP." -us,scenario_109,child3_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model treated being under age 19, below the CHIP income limit, and uninsured as jointly sufficient for CHIP eligibility. It skipped the Medicaid-eligibility exclusion: child3 qualifies under Florida's INFANT Medicaid category, so the child cannot qualify for CHIP." -us,scenario_109,child3_chip_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model conflated Medicaid infant eligibility with CHIP eligibility by grouping the programs into a single income-limit test. Child3's low income establishes Medicaid eligibility under the INFANT category, and that Medicaid eligibility makes the child ineligible for CHIP rather than eligible for it." -us,scenario_109,child3_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model collapsed Florida Medicaid and CHIP into one combined eligibility determination and treated qualification for either program as CHIP eligibility. Child3 specifically qualifies for Medicaid under the INFANT category, which triggers CHIP's exclusion for Medicaid-eligible children." -us,scenario_109,child3_early_head_start_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Early Head Start eligibility as requiring an explicit fact saying the child is enrolled or designated eligible. It ignored that the benchmark asks for eligibility under PolicyEngine rules, where age 0 plus household income below the Early Head Start income threshold determines eligibility." -us,scenario_109,child3_early_head_start_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly identified the under-3 age requirement but added nonexistent extra categorical requirements beyond the income pathway. It also failed to apply the income test decisively: a $30,000 household income for this five-person family is below the Early Head Start threshold, so TANF, SSI, homelessness, or foster-care status is not needed for eligibility." -us,scenario_109,child3_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 0 as sufficient for Head Start eligibility and never applied the Head Start income screen. It also conflated general child age-related program access with the preschool-age Head Start output, which PolicyEngine denies once household income exceeds the income threshold." -us,scenario_109,child3_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the Florida infant Medicaid MAGI pathway and treated Medicaid eligibility as requiring an unspecified additional qualifying indicator. Child3 is age 0, and 0.72 FPL is below Florida's 185% FPL threshold for infants, so the correct categorical and income tests make child3 eligible." -us,scenario_109,child3_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model identified the infant income pathway but then overrode it with nonexistent asset-test, Medicaid-expansion, enrollment, and CHIP-substitution conditions for this output. PolicyEngine's eligibility question asks current eligibility, not enrollment, and Florida's infant Medicaid MAGI category covers age-0 children with MAGI below 185% FPL." -us,scenario_109,child3_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated WIC eligibility as requiring an explicit provided fact that the child was WIC-enrolled or otherwise designated eligible. It failed to apply the WIC eligibility rule from the listed facts: Child 3 is age 0 and the five-person household's $30,000 income is below the 185% FPL WIC limit, so the child is eligible." -us,scenario_109,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated unused Child Tax Credit as if it could reduce federal income tax before refundable credits below zero, producing a negative liability. Nonrefundable credits are limited to pre-credit income tax liability, and any remaining CTC belongs in the refundable credit calculation rather than this output." -us,scenario_109,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived $8,231 of EITC and about $3,807 of refundable CTC, then improperly rounded their $12,038 sum to $12,000. The requested output was an annual amount, not a rounded estimate." -us,scenario_109,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly declared the household ineligible for EITC even though its own cited MFJ three-child phaseout endpoint exceeds the household's income. It also replaced the earned-income-based refundable CTC calculation with an unsupported $2,500 amount instead of $3,807.09." -us,scenario_109,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model explicitly calculated $8,231 of EITC plus $3,807 of refundable CTC, totaling $12,038, but submitted $10,379 without a computation supporting that number. Its final value discarded its own correct derivation." -us,scenario_109,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model computed refundable CTC of about $3,781 and EITC of about $8,046, then replaced the CTC with an unexplained $4,416 to reach $12,462. It also used net earnings after the 0.9235 adjustment directly for the refundable CTC rather than the $3,807.09 calculation used here." -us,scenario_109,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model imposed a large EITC reduction at $30,000, assigning only about $5,035 instead of the $8,231 three-child credit. It also used gross self-employment income in the 15% refundable CTC formula, producing $4,125 instead of $3,807.09." -us,scenario_109,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model repeatedly applied obsolete or mismatched EITC maxima and phaseout thresholds, reducing a full $8,231 credit. It also calculated refundable CTC from gross $30,000 self-employment income rather than the applicable earned-income measure, and its submitted $7,594 does not equal any total in its reasoning." -us,scenario_109,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated the $1,700-per-child refundable CTC cap as payable despite its own calculation showing that the 15% earned-income limit restricted the credit to about $3,807. It also reduced the EITC to $3,015 by applying the three-child schedule incorrectly instead of awarding $8,231." -us,scenario_109,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used $8,026 rather than the 2026 three-child EITC of $8,231 and imposed an unsupported $3,000 refundable CTC amount. The earned-income formula yields $3,807.09 of refundable CTC." -us,scenario_109,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model used an approximate $8,290 EITC instead of $8,231 and capped refundable CTC at an outdated $1,000 per child. The refundable CTC is governed by the earned-income formula here and equals $3,807.09, not $3,000." -us,scenario_109,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model reduced the three-child EITC to $3,588 even though the household receives $8,231 under the applicable schedule. It also understated the earned-income-based refundable CTC as $1,674 rather than $3,807.09." -us,scenario_109,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model approximated the EITC at $8,250 rather than using $8,231 and imposed a flat $3,000 Additional CTC. Applying the 15% earned-income formula produces $3,807.09 of refundable CTC." -us,scenario_109,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model approximated the EITC at $8,240 and used an outdated $1,000-per-child refundable CTC cap. The applicable components are $8,231 of EITC and $3,807.09 of refundable CTC." -us,scenario_109,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model understated the EITC by $12 and assigned a flat $3,000 refundable CTC. The earned-income calculation produces $3,807.09 rather than $1,000 per child." -us,scenario_109,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model used an estimated EITC maximum of $8,025.75 instead of the applicable $8,231. It also based refundable CTC on $27,705 of net earnings, whereas the applicable computation yields $3,807.09." -us,scenario_109,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The answer supplied no component calculation and overstated the combined credits by $1,361.91. The specified components sum to $8,231 of EITC plus $3,807.09 of refundable CTC, or $12,038.09." -us,scenario_109,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the explicitly listed $30,000 of self-employment income, which is earned income for both EITC and refundable CTC purposes. That income and the three qualifying children generate $8,231 of EITC and $3,807.09 of refundable CTC." -us,scenario_109,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model identified the correct $8,231 EITC but imposed an outdated $1,000-per-child refundable CTC amount. The 15% earned-income formula yields $3,807.09, raising the total to $12,038.09." -us,scenario_109,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model used gross $30,000 self-employment income in the refundable CTC formula, producing $4,125. The applicable earned-income computation after the self-employment-tax deduction yields $3,807.09." -us,scenario_109,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model calculated refundable CTC from $27,705, the 92.35% self-employment-tax base, and obtained about $3,781. The applicable earned-income amount reflects the half-self-employment-tax deduction and produces $3,807.09." -us,scenario_109,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model used gross $30,000 self-employment income to calculate refundable CTC as $4,125. Applying the correct earned-income measure produces $3,807.09 alongside the $8,231 EITC." -us,scenario_109,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model failed to recognize that $30,000 of self-employment income counts as earned income and that all three children qualify for the child-based credits. The household therefore receives both the $8,231 EITC and $3,807.09 refundable CTC." -us,scenario_109,federal_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model imposed obsolete post-TCJA rules: a $1,000-per-child refundable CTC cap and a $3,000 earnings threshold. The applicable refundable CTC is $3,807.09, and the applicable EITC is $8,231 rather than $8,226." -us,scenario_109,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model used an outdated $1,000-per-child refundable CTC cap, limiting the credit to $3,000 despite its own earned-income calculation exceeding that amount. It also approximated the EITC at $8,245 rather than applying the $8,231 value." -us,scenario_109,federal_refundable_credits,kimi-k2.6,llm_error,other,False,"The submitted $6,695 omits or sharply understates at least one of the two refundable-credit components. The correct component calculation is $8,231 of EITC plus $3,807.09 of refundable CTC." -us,scenario_109,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model reduced the EITC to $3,588 by applying the phaseout schedule incorrectly and treated refundable CTC as the full $5,100 per-child-cap total instead of enforcing the lower earned-income limit. Its submitted $4,288 also does not equal the $8,688 total stated in its reasoning." -us,scenario_109,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model incorrectly placed the household in the EITC phaseout region and substituted a conservative $5,475 estimate for the applicable $8,231 credit. It also used gross $30,000 self-employment income for refundable CTC, producing $4,125 instead of $3,807.09." -us,scenario_109,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of explicitly stated school-meal eligibility as disqualifying and failed to compute the household's SNAP receipt, which automatically confers free-meal eligibility. It also missed the independent income pathway: $30,000 is 78% of the applicable poverty guideline, below the 130% free-meal cutoff." -us,scenario_109,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated Florida as having an adult Medicaid expansion pathway for low-income adults. It applied a low-MAGI adult eligibility shortcut and missed that this 30-year-old head has no qualifying Florida Medicaid category, so the correct eligibility result is no despite income at 0.72 FPL." -us,scenario_109,head_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated the presence of a WIC-eligible infant as making the parent or caretaker categorically eligible. WIC categorical eligibility does not extend from an eligible child to the head; the head is a 30-year-old adult with no listed pregnancy, postpartum, or breastfeeding status, so the head fails the categorical test even though household income is below the limit." -us,scenario_109,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model converted the infant's categorical eligibility into household-member eligibility and counted the head as eligible because the head is the parent/applicant. WIC eligibility for this output applies to the individual, and a 30-year-old head with no listed pregnancy, postpartum, or breastfeeding status is outside the eligible WIC categories despite passing the household income test." -us,scenario_109,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied only the under-185% reduced-price ceiling and improperly treated rent as reducing school-meal income. It failed to apply the free-meal tier first: the 0.78 FPG ratio and categorical eligibility place the household in the mutually exclusive FREE tier, leaving no positive reduced-price support." -us,scenario_109,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly identified that income below 130% FPG qualifies the children for free meals, then incorrectly treated categorical eligibility as also producing reduced-price support. PolicyEngine assigns the superior FREE tier exclusively, so free-meal qualification makes reduced_price_school_meals_eligible equal 0." -us,scenario_109,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model stated the correct 92.35% net-earnings adjustment and 15.3% rate but multiplied them incorrectly. $30,000 × 0.9235 × 0.153 equals $4,238.865, not $4,242." -us,scenario_109,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model selected the correct net-earnings base and combined rate but its approximate arithmetic overstated the tax. Applying 15.3% to $27,705 produces $4,238.865, not $4,243." -us,scenario_109,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model explicitly wrote the correct expression but evaluated it incorrectly: $30,000 × 0.9235 × 0.153 is $4,238.865, not $4,234. Additional Medicare Tax does not alter this household's result and was irrelevant to the discrepancy." -us,scenario_109,self_employment_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model used the correct 92.35% base factor and 15.3% rate but incorrectly rounded the product to $4,241. The product is $4,238.865, yielding $4,238.86 in the engine calculation." -us,scenario_109,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model's component method leads to about $4,238.87, and its own reasoning even states a total near $4,238, but it submitted $4,233. Using unrounded components gives $3,435.42 of Social Security tax plus $803.445 of Medicare tax, totaling $4,238.865." -us,scenario_109,snap,claude-fable-5,llm_error,other,False,"The model's final $3,768 contradicts its own SNAP calculation of roughly $625–$626 per month and introduces an unsupported reduction for payroll or self-employment adjustments. SNAP applies the 20% earned-income deduction to the $2,500 monthly self-employment income and does not apply the unexplained reduction that halved its result." -us,scenario_109,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an unexplained $244 monthly benefit despite deriving net income near $1,830 and asserting a $1,316 maximum allotment; those inputs do not produce $244. It also improperly substituted tax AGI and a half-self-employment-tax adjustment into its SNAP reasoning instead of applying the traced SNAP deductions and monthly allotments." -us,scenario_109,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a $1,613 monthly maximum allotment for a five-person household, far above the applicable $1,183 January–September and $1,209.52 October–December amounts. That inflated allotment directly produced the $13,016 annual estimate." -us,scenario_109,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model deducted self-employment tax from SNAP income and then applied an unsupported shelter deduction even though $800 rent is below half of adjusted income. Those extra deductions understated net income and inflated the benefit to $11,436." -us,scenario_109,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model asserted a $424 monthly benefit without applying the five-person maximum allotments of $1,183 and $1,209.52 to net monthly income of $1,739 and $1,733.15. The traced expected contributions instead yield about $661.30 for nine months and $689.62 for three months." -us,scenario_109,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used estimated maximum allotments of $1,017–$1,034 instead of the applicable $1,183 and $1,209.52 amounts. Its estimated standard deduction was also below the traced $261 and $266.85 deductions, but the understated allotment caused most of the shortfall." -us,scenario_109,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model miscounted the five listed people as a household of six and then stated that $2,309 monthly income exceeded a roughly $4,590 gross limit, which reverses its own comparison. The actual five-person household passes the income tests and receives positive SNAP throughout the year." -us,scenario_109,snap,deepseek-v4-pro,llm_error,period_annualization,False,"The model applied one estimated standard deduction and one maximum allotment for all twelve months. It missed the October parameter update from a $261 deduction and $1,183 allotment to $266.85 and $1,209.52, so its flat $654 monthly annualization understated the annual total." -us,scenario_109,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model estimated net income as $1,750 and imposed a flat $695 monthly benefit without matching either effective parameter period. The traced calculation uses net income of $1,739 and $1,733.15 and produces about $661.30 for nine months and $689.62 for three months." -us,scenario_109,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $9,180 implies a flat $765 monthly allotment, but the model supplied no computation supporting that amount. Applying the earned-income and standard deductions followed by the effective maximum allotments yields about $661.30 for nine months and $689.62 for three months." -us,scenario_109,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,The model invoked an excess shelter deduction even though the $800 monthly rent is less than half of adjusted income in both parameter periods. That nonexistent deduction reduced countable income and inflated the estimate to $786 per month. -us,scenario_109,snap,gemini-3.5-flash,llm_error,period_annualization,False,The model used a single estimated net income and maximum allotment to annualize $665 for all twelve months. It missed the October changes in both the standard deduction and maximum allotment and failed to sum the unrounded monthly benefits. -us,scenario_109,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated the $800 monthly rent as producing a SNAP deduction, but rent does not exceed half of adjusted income and therefore generates no excess shelter deduction. That extra deduction inflated the annual benefit to $9,030." -us,scenario_109,snap,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $236 standard deduction and $1,155 maximum allotment instead of the effective values for each month. The applicable schedule is $261 and $1,183 through September, then $266.85 and $1,209.52 from October onward." -us,scenario_109,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model characterized the household as having high shelter costs and returned about $485 per month, which is consistent with applying an unsupported income or allotment adjustment. The $800 rent produces no excess shelter deduction, and the traced deductions and allotments yield $8,020.55 annually." -us,scenario_109,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model imposed a no-take-up assumption even though the prompt explicitly directs the model to assume program take-up when required. The household passes SNAP's income, asset, and work-requirement tests, so assigning zero without calculating eligibility was incorrect." -us,scenario_109,snap,gpt-5.5,llm_error,period_annualization,False,"The model correctly computed the January–September benefit near $661 but multiplied that amount by twelve. It failed to apply the October increase in the standard deduction and maximum allotment, which raises the final three monthly benefits to about $689.62." -us,scenario_109,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The $7,908 result implies a flat $659 monthly benefit and therefore does not apply the two parameter periods. SNAP is about $661.30 for January–September and $689.62 for October–December, summed without monthly rounding." -us,scenario_109,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model applied the January–September $661 result to all twelve months. It omitted the October updates to the standard deduction and maximum allotment, which increase each of the final three monthly benefits to about $689.62." -us,scenario_109,snap,gpt-5.6-terra,llm_error,period_annualization,False,The model used a flat estimated $659 monthly allotment across the year. It missed both the higher January–September amount generated by the traced parameters and the further October increase to about $689.62. -us,scenario_109,snap,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly declared the household over SNAP's income and asset thresholds. The five-person household's $2,500 monthly earned income passes the applicable tests after SNAP deductions, and its $900 bank balance does not disqualify it." -us,scenario_109,snap,grok-4.5,llm_error,period_annualization,False,"The model used projected parameters to produce a flat $690 monthly benefit for all twelve months. The applicable schedule yields about $661.30 for nine months and $689.62 for three months, so annualizing the later-period amount overstated the result." -us,scenario_109,snap,grok-build-0.1,llm_error,period_annualization,False,"The model substituted a single $254 deduction and $1,189 allotment for the two effective schedules. It needed to use $261 and $1,183 through September, then $266.85 and $1,209.52 from October through December, and sum unrounded monthly values." -us,scenario_109,snap,kimi-k2.6,llm_error,thresholds_rates,False,"The model used an estimated $300 monthly standard deduction and a projected $1,220 maximum allotment instead of the effective monthly parameters. Those inflated deductions and allotments produced a $710 monthly estimate above both traced benefit levels." -us,scenario_109,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly reproduced the January–September calculation but applied its rounded $661 result to all twelve months. It omitted the October increase to a $266.85 standard deduction and $1,209.52 maximum allotment and also failed to sum unrounded monthly benefits." -us,scenario_109,snap,minimax-m3,llm_error,other,False,"The model asserted net income near zero and a near-maximum allotment, then submitted $5,868 without reconciling that number to its own calculation. The $800 rent creates no excess shelter deduction, leaving net income of $1,739 before October and $1,733.15 afterward." -us,scenario_109,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated income after the 20% earned-income deduction as the gross-income-test amount and then effectively awarded the full estimated maximum allotment. SNAP must subtract the expected contribution equal to 30% of net income, producing about $661.30 for nine months and $689.62 for three months rather than $962 monthly." -us,scenario_109,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a general low-income adult Medicaid pathway in non-expansion Florida and treated the spouse's own unlisted income and low assets as sufficient. Florida adult Medicaid requires a qualifying category, and this spouse has medicaid_category NONE, so assets and individual income do not establish eligibility." -us,scenario_109,spouse_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly reasoned that the spouse fails Florida parent/caretaker Medicaid because household income exceeds the very low parent limit, then submitted the opposite binary value. Its value is a parse-reasoning contradiction: the explanation says the spouse is not eligible as a parent, but the output reports eligible." -us,scenario_109,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly identified that Florida parent/caretaker Medicaid is unavailable at the household's income level, then incorrectly converted the presence of an infant into adult eligibility. A 0-year-old child does not by itself establish pregnancy or postpartum Medicaid status for the spouse; with no pregnancy or postpartum fact listed, the spouse has no qualifying category." -us,scenario_109,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model inferred pregnancy or postpartum Medicaid coverage from the presence of a 0-year-old even though the prompt makes unlisted statuses false. PolicyEngine therefore treats the spouse as not pregnant or postpartum, and the spouse also exceeds Florida's parent/caretaker income pathway, leaving medicaid_category NONE." -us,scenario_109,spouse_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model applied a generic low-income adult Medicaid rule and ignored Florida's categorical restrictions for non-expansion adults. The spouse's MAGI level does not matter without an eligible category, and PolicyEngine assigns medicaid_category NONE." -us,scenario_109,spouse_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model inferred postpartum status from the presence of a newborn even though the prompt made unlisted statuses false. A 29-year-old spouse is not WIC-eligible merely because the household includes an infant; only the infant meets the child-under-5 WIC category. -us,scenario_109,spouse_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated living with a child under age 5 as a categorical WIC pathway for the adult spouse. WIC categorical eligibility covers pregnant, postpartum, or breastfeeding women and young children, and the spouse had no listed pregnancy, postpartum, or breastfeeding status." -us,scenario_109,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a caregiver or postpartum-adjacent WIC category from the spouse's relationship to the infant. PolicyEngine applies the categorical test to the person, and the spouse has no listed qualifying pregnancy, postpartum, or breastfeeding status." -us,scenario_109,spouse_wic_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model inferred that the spouse recently gave birth or was breastfeeding from the age-0 child, contrary to the instruction that unlisted statuses are false. Income below the WIC threshold does not create eligibility without the spouse's own categorical status." -us,scenario_109,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated the presence of a child under age 1 as making the spouse postpartum or breastfeeding and added adjunctive eligibility speculation. The spouse lacks any listed WIC categorical status, so the income test and possible Medicaid or SNAP links do not make the spouse eligible." -us,scenario_109,spouse_wic_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model assumed the spouse was the postpartum mother of the infant. The benchmark does not list pregnancy, postpartum, or breastfeeding status for the spouse, so the spouse fails WIC categorical eligibility despite household income below 185% FPL." -us,scenario_109,spouse_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model converted an infant in the household into likely postpartum or breastfeeding status for the spouse. Under the prompt, unlisted statuses are false, and WIC does not cover an adult spouse solely as an infant's household member." -us,scenario_109,spouse_wic_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model inferred recent postpartum or breastfeeding status from the spouse being a female adult with an infant. Those statuses were not listed and therefore are false in the benchmark, leaving the spouse without the required WIC categorical eligibility." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model included the $6,323 estate-income input in AGI and used a guessed $16,100 standard deduction, producing $153,843 of taxable income. It omitted the deduction computation totaling $17,752.06, including the $1,264.59 qualified business income deduction, that reduces taxable income to $145,669.58; its submitted $21,120 also contradicts its own $25,840.76 calculation." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model included estate income in gross income, deducted the IRA contribution directly, and used an incorrect $15,000 standard deduction instead of the traced AGI and $17,752.06 deduction total. It also invented an unexplained $1,300 nonrefundable-credit reduction despite stating no applicable credit." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model included $6,323 of estate income in AGI and substituted an approximate $16,100 standard deduction for the traced $17,752.06 deduction total. It therefore taxed $153,843 rather than $145,669.58 and missed the $1,264.59 qualified business income deduction." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI, allowed a $198 IRA deduction using contradictory reasoning, and used an approximate $16,100 standard deduction. The correct computation has $163,421.64 of AGI and $17,752.06 of deductions, including the QBI deduction, so its ordinary taxable-income base was overstated." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model first overstated taxable income by including estate income and omitting the full $17,752.06 deduction computation. It then submitted $30,150 even though its stated ordinary-plus-preferential calculation was approximately $26,133, so the final value does not follow from its own arithmetic." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI and estimated a $15,750 standard deduction rather than applying the traced $17,752.06 deductions, including the $1,264.59 QBI deduction. This left ordinary taxable income about $8,523 too high and consequently overstated ordinary-rate tax." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model incorrectly included estate income and treated unreimbursed employee expenses as deductible, but its own resulting calculation still produced about $26,045. It then submitted $20,200 based on unspecified “adjustments,” so the answer discards its own tax calculation and never applies the traced $17,752.06 deduction total." -us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly deducted employer health premiums from wages, restored unreimbursed-employee deductions and a personal exemption, and then applied obsolete 10%–28% ordinary brackets. It also effectively taxed preferential income twice by reporting $32,490 of ordinary tax on a base that already included the income later taxed at 15%." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI and applied both a restored personal exemption and a miscellaneous-itemized deduction for unreimbursed employee expenses. The traced computation instead uses $163,421.64 of AGI and $17,752.06 of taxable-income deductions, including QBI, with no personal-exemption shortcut." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer omits estate income from its listed inputs but provides no taxable-income split or deduction arithmetic capable of producing $19,688. Applying the traced $17,752.06 deductions gives $145,669.58 of taxable income and $23,897.44 of tax, so its number reflects an excessive unreported deduction or an understated ordinary-rate computation." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model deducted employer-sponsored health premiums from wages and applied sunset-era itemized deductions and personal exemptions. The computation instead starts with $95,755.10 of employment income after only the pre-tax 401(k) amount and uses $17,752.06 of deductions, including QBI." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI and invented state-income-tax, unreimbursed-employee, and personal-exemption deductions totaling far beyond the traced deduction calculation. It also omitted the $1,264.59 QBI deduction pathway while using an obsolete post-sunset framework." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted employer health-insurance premiums from wages, reducing AGI below the traced $163,421.64. It then supplied no deduction or ordinary/preferential tax arithmetic supporting $22,415; the traced bases yield $17,764.88 of ordinary tax plus $6,132.56 of preferential tax." -us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI and used an estimated $15,400 standard deduction. It missed the traced $17,752.06 total deductions, including the $1,264.59 QBI deduction, and therefore overstated both taxable income and ordinary-rate tax." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The explanation explicitly states that federal income tax remains positive, but the model submitted zero. The traced ordinary and preferential components are $17,764.88 and $6,132.56, and no nonrefundable credit eliminates them." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model gives no reproducible taxable-income or rate calculation and refers generically to medical and itemized assumptions. The traced $145,669.58 taxable income produces $23,897.44, so $16,811 embodies an excessive deduction, an omitted income component, or failure to stack preferential income correctly." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model included estate income in its $169,943 AGI and used a $16,100 standard deduction. It omitted the traced $17,752.06 deduction calculation, including $1,264.59 of QBI deduction, leaving taxable income $8,173.42 too high." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model relied on a $387 nonitemizer charitable deduction but omitted the traced $1,264.59 qualified business income deduction and did not reproduce the $17,752.06 total deduction. Its resulting $25,611 therefore uses the wrong taxable-income base instead of $145,669.58." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI and applied an estimated $16,100 standard deduction. It missed the full $17,752.06 deduction computation, including the $1,264.59 QBI deduction, and consequently overstated ordinary taxable income." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used $169,943 of AGI and a $16,100 standard deduction rather than the traced $163,421.64 AGI and $17,752.06 deductions. That shortcut omits the $1,264.59 QBI deduction and overstates taxable income from $145,669.58 to $153,843." -us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model supplied only a broad estimate based on unspecified itemized deductions and “standard nonrefundable credits,” although no such credit reduces the traced liability. The required derivation uses $145,669.58 of taxable income and splits tax into $17,764.88 ordinary and $6,132.56 preferential components." -us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI and applied restored unreimbursed-employee deductions and a personal exemption. The traced computation instead uses $17,752.06 of deductions including QBI, not the model's obsolete post-TCJA-sunset deduction framework." -us,scenario_110,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model deducted the IRA contribution, invented an Ohio-income-tax itemization, allowed unreimbursed employee expenses and a personal exemption, and applied obsolete 10%/15%/25% brackets. The traced calculation uses $145,669.58 of taxable income under the applicable main rates and separately taxes $40,883.77 of preferential income." -us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no output or explanation for the requested variable. -us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model invented an $8,425.50 overtime deduction from the straight-time hourly wage and hours, even though gross wages were already supplied as the complete annual total and no overtime-pay input supported that deduction. It also used a separate charitable deduction and failed to use the traced $17,752.06 deduction total, including QBI." -us,scenario_110,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model asserted that the calculation returned zero without performing the income, deduction, or rate computation. The traced ordinary tax of $17,764.88 and preferential tax of $6,132.56 remain due because no nonrefundable credits offset them." -us,scenario_110,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model omitted the short-term gain and pre-tax 401(k) reduction from its AGI arithmetic and used an estimated standard deduction instead of the traced deduction total. More decisively, it calculated $26,973 in its explanation but submitted $43,879.20, a value unsupported by any stated computation." +us,scenario_108,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model applied the ordinary 100% FPL net-income test as a disqualifying condition and omitted categorical eligibility through TANF non-cash assistance. That pathway preserves eligibility and triggers the minimum allotment despite net income of $2,035.67." +us,scenario_108,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model disqualified the household under a 130% gross-income limit and omitted Wisconsin categorical eligibility through TANF non-cash assistance. The categorically eligible household receives the minimum allotment rather than zero. +us,scenario_108,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly reached a negative formula allotment and identified the minimum-benefit rule, but then abandoned its own approximately $23-per-month result and submitted $2,227. The applicable monthly minimums sum to $287.68, not an unexplained larger allotment." +us,scenario_108,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated failure of the ordinary 100% FPL net-income test as disqualifying and omitted categorical eligibility through TANF non-cash assistance. It then submitted $2,376 despite explicitly concluding that the benefit was zero; the categorical pathway instead yields the annual minimum allotments totaling $287.68." +us,scenario_108,snap,claude-opus-5,llm_error,period_annualization,False,"The model identified the minimum-allotment outcome but annualized a flat assumed $23 monthly amount. PolicyEngine applies $23.84 in most months and $24.37 in months covered by updated guidelines, which sum to $287.68." +us,scenario_108,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Wisconsin categorical eligibility did not remove the ordinary gross and net income barriers. TANF non-cash assistance establishes categorical eligibility here, so the negative formula amount is replaced by the minimum allotment." +us,scenario_108,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions reduced net income to about $1,700 and then converted that unsupported figure into a $164 monthly benefit. The trace produces $2,035.67 of net monthly income, a negative regular allotment, and therefore only the minimum allotment totaling $287.68 annually." +us,scenario_108,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model equated a negative maximum-allotment calculation with a zero benefit. Categorical eligibility through TANF non-cash assistance keeps the household eligible, and the one-person minimum-allotment rule replaces the negative result." +us,scenario_108,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model used gross-income limits and a negative formula allotment to assign zero, omitting TANF non-cash categorical eligibility. Once eligible, this one-person household receives the minimum allotment even though 30% of net income exceeds the maximum allotment." +us,scenario_108,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The zero answer treats the ordinary income threshold as dispositive and omits categorical eligibility through TANF non-cash assistance. That pathway produces minimum monthly allotments totaling $287.68. +us,scenario_108,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model disqualified the household because gross and net income exceeded ordinary SNAP limits. TANF non-cash categorical eligibility overrides that shortcut, leaving the household eligible for the minimum allotment." +us,scenario_108,snap,gemini-3.5-flash,llm_error,period_annualization,False,"The model correctly recognized categorical eligibility and the minimum-benefit rule but annualized a flat $23 monthly estimate. The applicable minimums are $23.84 and $24.37 across different benefit months, summing to $287.68." +us,scenario_108,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model treated excess gross income and a negative formula amount as producing ineligibility and zero benefits. TANF non-cash categorical eligibility instead leads to the one-person minimum allotment. +us,scenario_108,snap,glm-5.2,llm_error,categorical_eligibility,False,The model stopped when maximum allotment minus 30% of net income became negative. It omitted categorical eligibility through TANF non-cash assistance and the minimum-allotment floor for an eligible one-person household. +us,scenario_108,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model assigned an unsupported $1,172 annual estimate without applying the traced allotment formula. Net monthly income of $2,035.67 makes the regular allotment negative, so categorical eligibility yields only the minimum monthly amounts totaling $287.68." +us,scenario_108,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assigned zero because income and deductions did not support a positive regular allotment, but it omitted TANF non-cash categorical eligibility. The eligible household receives the minimum allotment even without additional unlisted deductions." +us,scenario_108,snap,gpt-5.5,llm_error,categorical_eligibility,False,The model correctly recognized that the standard allotment formula was negative but incorrectly converted that result to zero. Categorical eligibility through TANF non-cash assistance activates the minimum-allotment floor. +us,scenario_108,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $103 monthly estimate overstates the deductions and bypasses the traced net-income result. Net income is $2,035.67 per month, making the regular allotment negative and leaving only the minimum allotments totaling $287.68." +us,scenario_108,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model treated failure of the ordinary elderly/disabled net-income limit as disqualifying. TANF non-cash categorical eligibility keeps the household eligible, after which the minimum-allotment rule applies." +us,scenario_108,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model used the one-person elderly/disabled net-income limit as an absolute eligibility bar. It omitted TANF non-cash categorical eligibility and therefore missed the minimum benefit. +us,scenario_108,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model relied exclusively on ordinary one-person income limits. TANF non-cash assistance establishes categorical eligibility, so the household receives minimum monthly allotments totaling $287.68." +us,scenario_108,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model compared net income with 100% FPL and treated the excess as disqualifying. That omits TANF non-cash categorical eligibility, which preserves eligibility and leads to the minimum allotment." +us,scenario_108,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model accurately described the ordinary net-income-test failure but incorrectly ended the calculation there. TANF non-cash categorical eligibility means that failure does not eliminate eligibility, and the negative regular allotment is floored at the minimum." +us,scenario_108,snap,inkling,llm_error,categorical_eligibility,False,The model treated net income above 100% FPL as dispositive. It omitted categorical eligibility through TANF non-cash assistance and the resulting minimum-allotment floor. +us,scenario_108,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model concluded that exceeding the ordinary elderly/disabled net-income limit caused ineligibility. TANF non-cash categorical eligibility keeps the household eligible for the minimum allotment. +us,scenario_108,snap,kimi-k3,llm_error,categorical_eligibility,False,The model correctly found that 30% of net income exceeded the maximum allotment but incorrectly assigned zero. It omitted TANF non-cash categorical eligibility and the minimum allotment paid to this eligible one-person household. +us,scenario_108,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model used the gross and net poverty thresholds to deny SNAP and also omitted the 20% earned-income deduction from its stated net calculation. The decisive error is failure to apply TANF non-cash categorical eligibility, under which the household receives the minimum allotment." +us,scenario_108,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model repeatedly recalculated deductions, introduced an unlisted utility allowance, and ultimately equated a negative regular allotment with zero. TANF non-cash categorical eligibility and the one-person minimum-allotment rule produce $287.68 annually; the shelter deduction is based on the listed rent without inventing utility expenses." +us,scenario_108,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated excess countable income as an absolute bar to SNAP. It omitted TANF non-cash categorical eligibility, which leads to the minimum monthly allotments totaling $287.68." +us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $10,392 survivor-benefit amount as Wisconsin-taxable income and started from $18,048 of AGI. Excluding that amount leaves only $7,656 of taxable wages, which Wisconsin deductions and exemptions fully offset." +us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model's $18,048 AGI includes the $10,392 survivor-benefit input as taxable Wisconsin income. That benefit is not part of the Wisconsin taxable base, so the remaining $7,656 of wages is eliminated by Wisconsin deductions and exemptions." +us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Although the model applied a personal exemption and renter-related school property tax credit, it first overstated Wisconsin AGI by including the $10,392 survivor-benefit input. With that nontaxable amount excluded, deductions and exemptions eliminate the tax before the renter credit is needed." +us,scenario_108,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model explicitly added wages of $7,656 and survivor benefits of $10,392 to obtain $18,048 of taxable income. The $10,392 survivor-benefit input is not Wisconsin-taxable income, leaving only wages that are fully offset by the applicable deductions and exemptions." +us,scenario_108,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model explicitly treated the survivor-benefit income as taxable alongside wages. Excluding that $10,392 amount leaves $7,656 of Wisconsin taxable-source income before deductions, and the applicable deductions and exemptions reduce the liability to zero." +us,scenario_108,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model explicitly included the $10,392 survivor-benefit input in its $18,048 Wisconsin AGI and then applied the standard-deduction phaseout to that overstated base. Excluding the survivor benefits leaves only $7,656 of wages, which is fully sheltered by Wisconsin deductions and exemptions." +us,scenario_108,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required output contract." +us,scenario_108,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the separately listed $10,392 survivor-benefit amount in homestead household income, inflating it from $20,590 to $30,982. It then abandoned its own zero calculation and inserted an unsupported $636 estimate instead of applying the credit formula to $1,440 of property-tax equivalent." +us,scenario_108,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly excluded Wisconsin's refundable homestead credit from state refundable income-tax credits by labeling it merely property-tax relief. The eligible elderly and disabled renter receives that refundable credit through the income-tax system, producing $271.39." +us,scenario_108,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model excluded the $12,934 of tax-exempt Social Security from homestead household income and substituted the separate $10,392 survivor-benefit figure, yielding an incorrect $18,048 income base. The required income measure is $20,590, and the exact phaseout calculation yields $271.39 rather than the rounded $300 estimate." +us,scenario_108,state_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model double-counted the $10,392 survivor-benefit amount alongside wages and Social Security, inflating homestead income to $30,982. Wisconsin homestead income is $20,590 here, which remains within the credit calculation and yields $271.39." +us,scenario_108,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model used an approximate $18,000 income figure and only estimated the homestead-credit phaseout. Using the traced $20,590 homestead income and $1,440 property-tax equivalent produces $271.39, not $371." +us,scenario_108,state_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model double-counted the separately listed $10,392 survivor-benefit amount, raising homestead income from $20,590 to $30,982 and incorrectly applying the income cutoff. The correct income base preserves eligibility and yields a $271.39 homestead credit." +us,scenario_108,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model overlooked the Wisconsin homestead-credit pathway for an elderly or disabled renter and wrongly treated the absence of dependents as dispositive. At age 85 with disability, $7,200 of rent, and $20,590 of homestead income, the claimant qualifies for $271.39." +us,scenario_108,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used $30,982 as homestead income by adding the separate $10,392 survivor-benefit amount to wages and Social Security. The applicable income is $20,590, so the homestead credit is $271.39." +us,scenario_108,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model inflated Wisconsin homestead income to $30,982 by counting the separate survivor-benefit amount in addition to wages and Social Security. The traced income base is $20,590, which generates a $271.39 credit." +us,scenario_108,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model asserted that no refundable-credit eligibility existed without applying Wisconsin's homestead credit for elderly or disabled renters. The claimant's age, disability, rent, and $20,590 homestead income produce $271.39." +us,scenario_108,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model double-counted the separately listed $10,392 survivor-benefit amount and therefore compared an erroneous $30,982 income figure with the homestead limit. The correct $20,590 income base yields $271.39." +us,scenario_108,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $30,982 as homestead household income by adding both survivor-benefit entries. Wisconsin uses $20,590 in this calculation, leaving the claimant eligible for $271.39." +us,scenario_108,state_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model overstated homestead income as $30,982 by adding the separate $10,392 survivor-benefit amount to wages and Social Security. With the correct $20,590 income, the credit equals $271.39." +us,scenario_108,state_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model double-counted the survivor-benefit information and incorrectly concluded that $30,982 exceeded the homestead limit. The homestead-income measure is $20,590, and the resulting refundable credit is $271.39." +us,scenario_108,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to recognize the Wisconsin homestead credit triggered by the claimant's elderly and disabled status and rental costs. Applying that pathway to $20,590 of household income and $1,440 of property-tax equivalent yields $271.39." +us,scenario_108,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model focused on an EITC-like credit and omitted Wisconsin's refundable homestead credit. The elderly, disabled renter meets the homestead pathway and receives $271.39." +us,scenario_108,state_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model treated the separately listed $10,392 survivor-benefit amount as additional homestead income and therefore incorrectly exceeded the income limit. Homestead income is $20,590, and the claimant receives $271.39." +us,scenario_108,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model wrongly made the absence of qualifying children dispositive and omitted the refundable Wisconsin homestead credit. An 85-year-old disabled renter can qualify independently of children, and this household receives $271.39." +us,scenario_108,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model placed the household above the homestead-credit income limit by using the wrong income aggregation. The correct homestead income is $20,590, which produces a $271.39 credit." +us,scenario_108,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model overlooked that age 85, disability, and rent establish the relevant Wisconsin homestead-credit pathway for this one-adult household. The credit calculation yields $271.39." +us,scenario_108,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model omitted the Wisconsin homestead credit without testing the elderly or disabled renter pathway. The listed facts qualify the household for a $271.39 refundable credit. +us,scenario_108,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted survivor-benefit information and used roughly $30,982 as homestead income, incorrectly eliminating the credit. The applicable $20,590 income produces $271.39; the lack of Wisconsin EITC does not eliminate the homestead credit." +us,scenario_108,state_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model formed $30,982 by adding the separate $10,392 survivor-benefit amount to wages and Social Security and then applied an estimated cutoff. The correct homestead income is $20,590, yielding $271.39." +us,scenario_108,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model correctly rejected Wisconsin EITC but omitted the refundable homestead credit. The elderly, disabled renter qualifies for $271.39 even without qualifying children." +us,scenario_108,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required output contract." +us,scenario_108,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model double-counted the separate $10,392 survivor-benefit amount and therefore used about $30,982 as broad household income. Wisconsin homestead income is $20,590 here, so the credit is $271.39." +us,scenario_108,state_refundable_credits,minimax-m3,llm_error,other,False,"The model incorrectly conditioned a refundable credit on positive Wisconsin income-tax liability. The homestead credit is refundable even when pre-credit state income tax is zero, and this household receives $271.39." +us,scenario_108,state_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted the $10,392 survivor-benefit amount alongside $12,934 of Social Security and $7,656 of wages, producing an erroneous $30,982 homestead-income figure. The correct $20,590 income remains eligible and yields $271.39." +us,scenario_108,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the absence of qualifying children as eliminating all Wisconsin refundable credits and omitted the homestead credit. The elderly, disabled renter qualifies for $271.39 independently of child-related credits." +us,scenario_109,child1_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model checked only CHIP's age and upper-income limits and treated satisfying them as sufficient. It omitted the prior Medicaid screen: the age-9 child qualifies for Florida Medicaid in the OLDER_CHILD category, which makes the child ineligible for CHIP." +us,scenario_109,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model calculated MAGI and compared it only with CHIP's upper-income limit. It failed to test the lower Medicaid eligibility pathway first; at this income, the age-9 child is Medicaid-eligible in Florida and therefore excluded from CHIP." +us,scenario_109,child1_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model recognized that CHIP applies between Medicaid and CHIP income limits but incorrectly placed this household in that interval. The age-9 child remains within Florida's Medicaid limit for the OLDER_CHILD category, so the child never enters the CHIP coverage band." +us,scenario_109,child1_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model treated being under age 19, uninsured, and below 200% FPL as a complete CHIP test. It omitted CHIP's exclusion of Medicaid-eligible children; this child qualifies for Florida Medicaid under the OLDER_CHILD category and is consequently not CHIP-eligible." +us,scenario_109,child1_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model compared the household's estimated FPL percentage only with Florida CHIP's upper limit. It failed to apply the Medicaid-first eligibility screen, under which the age-9 child qualifies as an OLDER_CHILD and is excluded from CHIP." +us,scenario_109,child1_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"The answer equated income being within a broad CHIP-related range with CHIP eligibility. The correct sequence first establishes that the age-9 child is Medicaid-eligible in Florida's OLDER_CHILD category, and that Medicaid eligibility bars CHIP eligibility." +us,scenario_109,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated eligibility as requiring an explicit qualifying indicator instead of applying Florida's Medicaid older-child pathway. Child 1 is age 9 and the household's MAGI is 72% FPL, below the older-child limit of 133% FPL, so the model should have returned Yes." +us,scenario_109,child2_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model stated that CHIP covers children who are not eligible for Medicaid but never performed that prerequisite test. Child2 qualifies for Medicaid under the OLDER_CHILD category, which automatically excludes the child from CHIP regardless of satisfying CHIP age or income limits." +us,scenario_109,child2_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model compared its estimated MAGI only with CHIP's upper income limit and omitted Medicaid eligibility. Child2 qualifies for Medicaid under the OLDER_CHILD category, and that eligibility precludes CHIP." +us,scenario_109,child2_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,The model treated being under age 19 and below Florida's CHIP income ceiling as sufficient. It failed to apply the Medicaid-exclusion condition: child2 is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP. +us,scenario_109,child2_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model combined the CHIP and Medicaid income ranges as though qualification for either supported CHIP eligibility. At its own estimate of about 82% FPL, child2 qualifies for Medicaid under the OLDER_CHILD category, and Medicaid eligibility makes the child ineligible for CHIP." +us,scenario_109,child2_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model applied only the under-19 rule and CHIP's 200% FPL ceiling. It omitted the prior Medicaid eligibility test: child2 qualifies under the OLDER_CHILD Medicaid category, which excludes the child from CHIP." +us,scenario_109,child2_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"The model used a generic CHIP age-and-income shortcut without testing Medicaid eligibility. The correct eligibility sequence places child2 in the OLDER_CHILD Medicaid category first, so CHIP eligibility is false under the programs' mutual-exclusivity rule." +us,scenario_109,child2_head_start_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly extended preschool Head Start eligibility to a six-year-old and treated $30,000 as below the applicable income limit. Child 2 fails the age rule, and the household income exceeds the Head Start threshold used for this household." +us,scenario_109,child2_head_start_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model correctly reasoned that age 6 is outside the preschool Head Start range but submitted value 1 instead of 0. Its numeric answer directly contradicts its stated eligibility conclusion. +us,scenario_109,child2_head_start_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model wrongly treated a six-year-old as within Head Start’s preschool-age range and improperly reduced self-employment income using the 92.35% self-employment-tax adjustment for the Head Start income test. The applicable household income remains approximately $30,000 and exceeds the program threshold." +us,scenario_109,child2_head_start_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model acknowledged that age 6 is technically too old but nevertheless converted an unsupported possible exception into eligibility. With no fact establishing a qualifying age exception, child 2 is outside the preschool Head Start category, and the income test also fails." +us,scenario_109,child2_head_start_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted that household income was low enough without applying the actual Head Start income threshold and ignored that child 2 is age 6. The approximately $30,000 household income exceeds the applicable threshold, and the child is outside the preschool-age range." +us,scenario_109,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required an unspecified additional qualifying indicator and treated positive household income as disqualifying. Child2's age and dependent status place the child in Florida's older-child Medicaid category, and 0.72 × FPL is below its 133% FPL limit." +us,scenario_109,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used an incorrect 100% FPL limit for a 6-year-old instead of Florida's 133% FPL older-child Medicaid threshold, then incorrectly imported adult non-expansion and asset-test restrictions into the child's MAGI pathway. At 0.72 × FPL, child2 qualifies for Medicaid rather than being routed exclusively to CHIP." +us,scenario_109,child2_wic_eligible,minimax-m3,llm_error,age_disability,False,"The model identified that age 6 exceeds WIC's child age limit but reversed the consequence and submitted eligibility anyway. PolicyEngine's WIC child category covers children under age 5, so Child 2 is categorically ineligible." +us,scenario_109,child2_wic_eligible,qwen3.8-max,llm_error,age_disability,False,"The model incorrectly treated a six-year-old as within WIC's child age range. WIC eligibility for children ends at age 5, so income eligibility cannot make Child 2 eligible." +us,scenario_109,child3_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model treated infancy, lack of reported coverage, and CHIP's general age range as sufficient for eligibility. It omitted the rule that a child already eligible for Medicaid is ineligible for CHIP; child3 qualifies for Medicaid under Florida's INFANT category." +us,scenario_109,child3_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model applied a CHIP income ceiling directly and called the infant's low income a CHIP-funded coverage range. It failed to evaluate Medicaid first: the same age and income qualify child3 for Medicaid under the INFANT category, which excludes the child from CHIP." +us,scenario_109,child3_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model treated being under 19, below the CHIP ceiling, and without listed coverage as sufficient conditions. It skipped the Medicaid-exclusion step: child3 is Medicaid-eligible under the INFANT category and therefore not CHIP-eligible." +us,scenario_109,child3_chip_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model conflated CHIP eligibility with the combined CHIP/Medicaid infant income limits and concluded that falling below either program's ceiling establishes CHIP eligibility. Child3's low income instead establishes Medicaid eligibility under the INFANT category, and that Medicaid eligibility categorically bars CHIP." +us,scenario_109,child3_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model merged Florida Medicaid and CHIP into a single eligibility conclusion. The correct sequencing assigns child3 to Medicaid's INFANT category at this income, after which the rule limiting CHIP to children who do not qualify for Medicaid returns no CHIP eligibility." +us,scenario_109,child3_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"The model used the shortcut that child status plus income within a CHIP-related limit implies CHIP eligibility. It omitted child3's Medicaid INFANT-category determination, which makes the child categorically ineligible for CHIP." +us,scenario_109,child3_early_head_start_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicit Early Head Start eligibility indicator as disqualifying instead of deriving eligibility from the supplied age and income facts. Child 3 is age 0 and the five-person household's $30,000 income passes the benchmark's income threshold." +us,scenario_109,child3_early_head_start_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly identified that Child 3 meets the age rule and that $30,000 is below its own estimated poverty threshold, but then incorrectly required an additional categorical condition such as TANF, SSI, homelessness, or foster care. Those conditions provide alternative categorical eligibility pathways; they are not required after the income pathway is satisfied." +us,scenario_109,child3_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 0 as within the preschool-age Head Start range, conflating Head Start with Early Head Start. It also treated age alone as sufficient and never applied the household income ceiling, under which $30,000 exceeds the stated $27,750 threshold." +us,scenario_109,child3_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored Florida's infant Medicaid pathway and incorrectly required unspecified additional qualifying indicators. Child3 is categorically eligible as an infant under age 1, and MAGI at 72% of FPL is below the 185% infant threshold." +us,scenario_109,child3_medicaid_eligible,qwen-3.7-max,llm_error,asset_resource,False,"The model correctly found income below an infant Medicaid threshold but then wrongly imposed an asset test and other categorical conditions on the MAGI infant pathway. Florida's lack of adult Medicaid expansion, CHIP availability, and confirmed enrollment are irrelevant because this age-0 dependent qualifies for Medicaid under the infant category at 72% of FPL." +us,scenario_109,child3_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly listed WIC status as disqualifying instead of computing eligibility from the supplied facts. The age-zero child satisfies WIC's child category, and the five-person household's income is below the 185%-of-poverty income limit, so the correct eligibility output is Yes." +us,scenario_109,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used an overstated projected standard deduction, incorrectly concluded taxable income and pre-credit tax were zero, and then subtracted an invented $284 of “unused” CTC. The correct computation produces income tax that uses $2,792.91 of the $6,600 CTC as a nonrefundable offset, while the remaining $3,807.09 is refundable; unused nonrefundable credits never make this output negative." +us,scenario_109,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived both components correctly—$8,231 of EITC and about $3,807 of refundable CTC—but replaced their $12,038.09 sum with an unjustified rounded value of $12,000." +us,scenario_109,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly declared the household ineligible for EITC even though $30,000 of self-employment income is below the married three-child phaseout start and yields the full $8,231 credit. It also replaced its own refundable CTC calculation of about $3,878 with an unsupported $2,500." +us,scenario_109,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model explicitly recomputed the correct components, $8,231 and $3,807, and correctly summed them to $12,038, but then submitted $10,379. The submitted value contradicts its completed calculation." +us,scenario_109,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model first calculated refundable CTC of about $3,781 from net self-employment earnings, then substituted an unsupported $4,416. It also used an outdated $8,046 EITC instead of the 2026 maximum of $8,231." +us,scenario_109,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model wrongly phased the three-child EITC down to about $5,035 at $30,000 of income instead of applying the $8,231 maximum. It also used $4,125 of refundable CTC without the self-employment adjustment that produces $3,807.09." +us,scenario_109,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model used outdated and internally inconsistent EITC phase-in and phaseout parameters, reducing a maximum $8,231 EITC. It also calculated ACTC from the full $30,000 rather than the applicable self-employment earned-income amount, then submitted $7,594 despite reasoning to several different totals." +us,scenario_109,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model reduced the EITC to about $3,015 even though the household receives the $8,231 maximum. It also treated the $5,100 per-child aggregate cap as payable despite acknowledging that the 15% earned-income limitation produces only about $3,807." +us,scenario_109,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model understated both components, using $8,026 rather than the $8,231 EITC and imposing an unsupported $3,000 refundable CTC amount instead of the $3,807.09 earned-income formula result." +us,scenario_109,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model imposed an obsolete $1,000-per-child ACTC cap, limiting refundable CTC to $3,000 instead of applying the earned-income formula that yields $3,807.09. It also overstated the 2026 EITC maximum as about $8,290 rather than $8,231." +us,scenario_109,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model reduced the EITC to $3,588 instead of awarding the $8,231 maximum for three qualifying children. Its $1,674 refundable CTC also fails to apply the 15% earned-income-above-$2,500 calculation, which yields $3,807.09." +us,scenario_109,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model approximated the EITC as $8,250 instead of using $8,231 and imposed an unsupported $3,000 ACTC amount. The refundable CTC earned-income limitation yields $3,807.09." +us,scenario_109,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model imposed an obsolete $1,000-per-child ACTC maximum, producing $3,000 rather than $3,807.09. It also estimated the EITC at $8,240 instead of applying the $8,231 parameter." +us,scenario_109,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model used $3,000 for refundable CTC rather than the $3,807.09 amount produced by the earned-income formula. It also understated the maximum three-child EITC by $12." +us,scenario_109,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an incorrect 2026 EITC maximum of $8,025.75 instead of $8,231. It also used $27,705 as the ACTC earned-income base, producing $3,780.75 rather than the traced $3,807.09." +us,scenario_109,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model supplied only a broad child-credit estimate and overstated the combined credits by $1,361.91. The actual computation is $8,231 of EITC plus $3,807.09 of refundable CTC, not $13,400." +us,scenario_109,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the explicitly listed $30,000 of self-employment income as earned income. That income supports both the $8,231 EITC and $3,807.09 refundable CTC." +us,scenario_109,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model correctly used the $8,231 EITC but imposed an obsolete $1,000-per-child ACTC amount. Applying the 15% earned-income limitation produces $3,807.09 of refundable CTC, not $3,000." +us,scenario_109,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model calculated refundable CTC as 15% of the full $30,000 minus $2,500, yielding $4,125. The applicable self-employment earned-income measure incorporates the half-SE-tax adjustment and yields $3,807.09." +us,scenario_109,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used net earnings of $27,705 directly for refundable CTC and obtained about $3,781. The traced ACTC earned-income base produces $3,807.09, which combined with the $8,231 EITC totals $12,038.09." +us,scenario_109,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model applied the ACTC formula to the full $30,000, producing $4,125. Using the applicable self-employment earned-income amount after the relevant adjustment produces $3,807.09." +us,scenario_109,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model omitted both refundable-credit pathways despite three qualifying children and $30,000 of self-employment earned income. Those facts generate an $8,231 EITC and $3,807.09 refundable CTC." +us,scenario_109,federal_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model imposed post-TCJA assumptions of a $3,000 threshold and a $1,000-per-child refundable cap, limiting ACTC to $3,000. The applicable refundable CTC calculation uses the $2,500 threshold and yields $3,807.09; the EITC maximum is $8,231." +us,scenario_109,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model used a $3,000 ACTC threshold and an obsolete $1,000-per-child refundable cap, producing only $3,000. It also overstated the EITC maximum as $8,245 rather than $8,231." +us,scenario_109,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied the refundable CTC calculation closely but used an EITC maximum of about $8,271. The 2026 maximum is $8,231, so the two components total $12,038.09 rather than $12,078." +us,scenario_109,federal_refundable_credits,kimi-k2.6,llm_error,other,False,"The submitted $6,695 does not equal either refundable component or their sum. The required derivation adds the $8,231 EITC and $3,807.09 refundable CTC to obtain $12,038.09." +us,scenario_109,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly phased the EITC down to $3,588 rather than awarding the $8,231 maximum. It also treated refundable CTC as the full $5,100 per-child-cap total instead of applying the binding earned-income limitation of $3,807.09, and its submitted $4,288 contradicts even those erroneous components." +us,scenario_109,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model used incorrect EITC phaseout thresholds and ultimately substituted a conservative $5,475 estimate for its own parameter calculations, instead of applying the $8,231 maximum. It also used full $30,000 self-employment income for ACTC, producing $4,125 rather than $3,807.09." +us,scenario_109,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the Child Tax Credit as entirely nonrefundable and omitted the Additional Child Tax Credit. The household receives $3,807.09 of refundable CTC in addition to the $8,231 EITC." +us,scenario_109,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of explicitly stated school-meal receipt as ineligibility instead of calculating eligibility from the household facts. It missed both automatic categorical qualification through $8,020.55 of annual SNAP benefits and direct income qualification at 78% of the federal poverty guideline, below the 130% free-meal limit." +us,scenario_109,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated Florida as offering Medicaid expansion to low-income adults and therefore used low MAGI as sufficient for eligibility. Florida has no expansion pathway for this age-30 head, and the head fits no other categorical pathway, producing medicaid_category NONE and ineligibility." +us,scenario_109,head_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model incorrectly transferred the infant's WIC categorical eligibility to the Head by treating parent or caretaker status in a WIC-eligible family as a qualifying category. WIC has no general parent/caretaker pathway; the Head lacks every listed adult categorical status and is therefore ineligible. +us,scenario_109,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated the infant's presence and the Head's role as parent/applicant as making the Head categorically eligible. Eligibility attaches separately to each person, and being an infant's parent or applicant does not qualify a 30-year-old adult who is not listed as pregnant, postpartum, or breastfeeding." +us,scenario_109,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied only an under-185%-of-poverty test and incorrectly treated housing costs as a school-meal income deduction. It failed to apply the free-tier precedence rule: at 78% of the poverty guideline and with categorical eligibility, the household receives free meals, which makes reduced-price support zero." +us,scenario_109,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly identified that income below 130% of poverty qualifies the children for free meals, but then incorrectly treated free-tier qualification as simultaneous categorical eligibility for positive reduced-price support. PolicyEngine assigns the mutually exclusive FREE tier, so reduced-price school-meal eligibility is false." +us,scenario_109,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model identified the correct 92.35% earnings base and 15.3% rate but multiplied them incorrectly. $30,000 × 0.9235 × 0.153 equals $4,238.865, not $4,242." +us,scenario_109,self_employment_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model used the correct net-earnings factor and combined rate but replaced the exact calculation with an inaccurate approximation. The stated formula yields $4,238.865, not about $4,243." +us,scenario_109,self_employment_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model correctly settled on applying 15.3% to 92.35% of income but evaluated $30,000 × 0.9235 × 0.153 incorrectly. That product is $4,238.865, not $4,234; no Additional Medicare Tax adjustment changes this result." +us,scenario_109,self_employment_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model stated the correct base and rate but performed an unsupported rounding to $4,241. The calculation produces $4,238.865 and therefore $4,238.86 in the benchmark output." +us,scenario_109,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model correctly calculated the $27,705 net self-employment earnings base, but rounded the Social Security and Medicare components prematurely and then submitted $4,233 even though its own rounded components sum to $4,238. Computing the combined rate without premature rounding gives $4,238.865." +us,scenario_109,snap,claude-fable-5,llm_error,other,False,"The model’s stated calculation produces roughly $7,500, but it submitted $3,768 after introducing unspecified payroll, self-employment, and higher-income adjustments that SNAP does not apply in that manner. It also omitted the October fiscal-year parameter update." +us,scenario_109,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an unsupported $244 monthly benefit despite its own net-income calculation and maximum-allotment figure, so its subtraction does not reconcile arithmetically. It also used IRS AGI and self-employment-tax adjustments instead of the traced SNAP earned-income and standard deductions and omitted the October schedule update." +us,scenario_109,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a $1,613 maximum monthly allotment for a five-person household, far above the applicable $1,183 January–September and $1,209.52 October–December amounts. This inflated the annual benefit to $13,016." +us,scenario_109,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model improperly reduced self-employment income by a 7.65% tax adjustment and applied a shelter deduction even though $800 rent does not exceed half of adjusted income. Those deductions understated SNAP net income, and it also failed to apply the two fiscal-year schedules month by month." +us,scenario_109,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model asserted a $424 monthly benefit without deriving it from the applicable five-person maximum allotments and expected contributions. The traced calculation yields roughly $661.30 for January–September and $689.62 for October–December, not a flat $424." +us,scenario_109,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used maximum allotments of $1,017–$1,034 instead of the applicable $1,183 and $1,209.52 amounts. It also treated the entire year as one schedule rather than applying the October fiscal-year update." +us,scenario_109,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model counted six household members even though the household contains two adults and three children, for a total of five. It then contradicted its own thresholds by saying $2,309 exceeds a stated $4,590 gross limit and wrongly denied SNAP eligibility." +us,scenario_109,snap,deepseek-v4-pro,llm_error,period_annualization,False,"The model imposed one approximate standard deduction and one $1,184 maximum allotment for all twelve months. It missed the October increase from the $261 and $1,183 schedule to the $266.85 and $1,209.52 schedule." +us,scenario_109,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model’s $695 monthly amount does not follow from the applicable maximum allotments and expected contributions. The correct monthly results are roughly $661.30 through September and $689.62 from October, which must be summed rather than replaced by a flat approximation." +us,scenario_109,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $9,180 implies a flat $765 monthly benefit, which exceeds both monthly amounts produced by the applicable maximum-allotment and expected-contribution calculation. The model did not identify or apply the standard deduction, 30% contribution, and October parameter update." +us,scenario_109,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model invoked an excess-shelter deduction even though $800 monthly rent is below half of adjusted income and therefore produces no shelter deduction. Its resulting $786 monthly benefit overstates the amounts obtained from the applicable standard deductions, maximum allotments, and 30% contributions." +us,scenario_109,snap,gemini-3.5-flash,llm_error,period_annualization,False,The model used approximate net income and a flat $665 monthly allotment for all twelve months. It omitted the October changes to both the standard deduction and maximum allotment and failed to sum the unrounded monthly results. +us,scenario_109,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated the $800 monthly rent as producing a deduction, but rent is below half of adjusted income and yields no excess-shelter deduction. That understates net income and inflates the annual SNAP benefit." +us,scenario_109,snap,glm-5.2,llm_error,thresholds_rates,False,"The model used an approximate $236 standard deduction and $1,155 maximum allotment instead of the applicable monthly parameters. It also applied that single schedule across the year and missed the October fiscal-year update." +us,scenario_109,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer characterized the household as having high shelter costs and submitted only $5,816 without a supporting computation. The $800 monthly rent creates no excess-shelter deduction, and the applicable deductions and allotments yield $8,020.55 annually." +us,scenario_109,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The prompt explicitly requires program take-up and supplies all unlisted numeric inputs as zero, so the model’s refusal to compute and its no-take-up assumption violate the stated benchmark rules. The household passes the SNAP eligibility tests and receives a positive benefit." +us,scenario_109,snap,gpt-5.5,llm_error,period_annualization,False,The model correctly reproduced the January–September calculation but multiplied its rounded $661 amount by twelve. It failed to apply the October increases to the standard deduction and maximum allotment for the final three months and failed to sum unrounded benefits. +us,scenario_109,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The submitted $7,908 implies one $659 monthly amount for the entire year. SNAP uses the January–September schedule and the updated October–December schedule, whose unrounded monthly amounts sum to $8,020.55." +us,scenario_109,snap,gpt-5.6-sol,llm_error,period_annualization,False,The model applied the January–September $661 monthly result to all twelve months. It omitted the October fiscal-year increases and rounded before annualizing. +us,scenario_109,snap,gpt-5.6-terra,llm_error,period_annualization,False,The model used a flat $659 monthly allotment across the year. It did not apply the distinct January–September and October–December parameter schedules or sum their unrounded monthly benefits. +us,scenario_109,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly declared the household over SNAP income and asset limits. Monthly gross income is $2,500, assets are only $900, and the traced deductions leave the household eligible throughout the year." +us,scenario_109,snap,grok-4.5,llm_error,period_annualization,False,"The model projected one $1,213 maximum allotment and one $690 benefit for all twelve months. The applicable maximum is $1,183 through September and $1,209.52 thereafter, with corresponding monthly benefits that must be summed unrounded." +us,scenario_109,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $254 standard deduction and $1,189 maximum allotment, neither of which matches the applicable January–September or October–December parameters. It then annualized a rounded single-month result instead of calculating both schedule periods." +us,scenario_109,snap,inkling,llm_error,thresholds_rates,False,"The model substituted approximate values of $250 for the standard deduction and $1,190 for the maximum allotment. It also used one rounded $665 monthly result for the full year and missed the October parameter update." +us,scenario_109,snap,kimi-k2.6,llm_error,thresholds_rates,False,"The model used a $300 monthly standard deduction and a projected $1,220 maximum allotment instead of the applicable $261/$1,183 and $266.85/$1,209.52 schedules. These parameter errors inflated the benefit, and the model also failed to calculate the two periods separately." +us,scenario_109,snap,kimi-k3,llm_error,period_annualization,False,The model correctly calculated the January–September benefit but extended the rounded $661 amount across all twelve months. It omitted the October increases to the standard deduction and maximum allotment and therefore missed the higher final-quarter benefits. +us,scenario_109,snap,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model inconsistently stated that net income was approximately zero even though $2,500 monthly income minus the 20% earned-income deduction and standard deduction leaves $1,739 before October. Rent produces no excess-shelter deduction, so neither a zero-net-income premise nor the submitted $5,868 follows from SNAP rules." +us,scenario_109,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the household as receiving a near-maximum $962 monthly benefit after vaguely invoking shelter deductions. Rent creates no excess-shelter deduction, and countable net income generates expected contributions of $521.70 before October and $519.90 afterward." +us,scenario_109,snap,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model used a four-person maximum allotment even though the SNAP household contains five people. It also set countable net income to zero, but the earned-income and standard deductions leave $1,739 monthly through September and slightly less from October, with no excess-shelter deduction." +us,scenario_109,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a federal minimum Medicaid pathway for any household member with no individual income and few assets. Florida requires this nondisabled adult to satisfy a covered category, and Medicaid MAGI eligibility uses household income rather than treating the spouse’s unlisted personal income as an isolated $0 eligibility base." +us,scenario_109,spouse_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly calculated that household income exceeds Florida’s parent/caretaker limit and explicitly concluded that the spouse is not eligible, but then submitted value 1. No alternative Medicaid category applies to reverse its own eligibility analysis." +us,scenario_109,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model improperly converted the presence of an infant into pregnancy or postpartum eligibility. An age-zero child does not establish that the spouse is pregnant, postpartum, or the child’s mother, and caretaker status alone fails because household income exceeds Florida’s parent/caretaker threshold." +us,scenario_109,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model inferred recent pregnancy and postpartum coverage from the existence of a newborn despite the instruction that unlisted statuses are false. The spouse is not specified as pregnant or postpartum, and the model’s own calculation places household income above Florida’s parent/caretaker limit." +us,scenario_109,spouse_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated low income and the absence of disqualifying facts as sufficient for adult Medicaid eligibility. In non-expansion Florida, a nondisabled 29-year-old must qualify through a covered category, and this spouse satisfies none of the available pathways." +us,scenario_109,spouse_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model inferred that the spouse was postpartum solely because a newborn lives in the household. Postpartum status was unlisted and therefore false, so the spouse fails WIC's adult categorical requirement even though the household passes the income test." +us,scenario_109,spouse_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly treated living with a child under age five as categorical WIC eligibility for the spouse. That condition makes the qualifying child a potential WIC participant; it does not make an adult caregiver eligible without pregnancy, breastfeeding, or postpartum status." +us,scenario_109,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a caregiver or “postpartum-adjacent” categorical pathway based on the infant's presence. WIC has no such adult pathway: the spouse herself must be pregnant, breastfeeding, or postpartum, and all three unlisted statuses are false." +us,scenario_109,spouse_wic_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model converted the existence of a zero-year-old child into a finding that the spouse recently gave birth and was postpartum or breastfeeding. The prompt forbids that inference by setting unlisted statuses to false, so the spouse does not satisfy WIC's categorical requirement despite income below 185% of the poverty guideline." +us,scenario_109,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated having an infant as equivalent to being the infant's postpartum or breastfeeding mother and then added an unsupported adjunctive-eligibility rationale. An infant's presence and the household's income do not establish the spouse's required pregnancy, postpartum, or breastfeeding status." +us,scenario_109,spouse_wic_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model assumed the spouse was the postpartum mother of the infant. Because postpartum status is not listed, it is false under the prompt, leaving the spouse outside every adult WIC category." +us,scenario_109,spouse_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model inferred postpartum or breastfeeding status from the spouse's age and the infant's presence. Neither fact establishes the spouse's own qualifying status, and the prompt explicitly makes those unlisted statuses false." +us,scenario_109,spouse_wic_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model inferred recent postpartum status or current breastfeeding from the existence of a zero-year-old child. Those personal statuses were unlisted and therefore false, so the spouse fails categorical eligibility before the income threshold affects the result." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used $169,943 of AGI and only a projected $16,100 standard deduction, thereby including $6,323 of estate income in its AGI construction and omitting both the $198.35 above-the-line deduction and the $1,264.59 QBI deduction reflected in the trace. Its submitted $21,120 also contradicts its own computed $25,841 total." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model started from $169,745 of AGI and used a $15,000 deduction instead of PolicyEngine's $163,421.64 AGI and $17,752.06 taxable-income deductions, including the $1,264.59 QBI deduction. It then invented an unexplained $1,300 nonrefundable-credit reduction despite stating that no applicable credit was identified." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model included the $6,323 estate-income input in AGI, rejected the $198.35 adjustment, and omitted the $1,264.59 QBI deduction. Those choices inflated ordinary taxable income from the traced amount and produced excessive ordinary-rate tax." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used $169,745 of AGI and a $16,100 standard deduction, missing PolicyEngine's $163,421.64 AGI and total $17,752.06 taxable-income deduction, including $1,264.59 of QBI deduction. This overstated ordinary taxable income and therefore ordinary tax." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's own components imply roughly $26,133, not its submitted $30,150, so the final value contains an arithmetic or transcription error. Its underlying calculation also used $169,745 of AGI and omitted the traced $1,264.59 QBI deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model constructed $169,943 of AGI by including estate income and denying the $198.35 above-the-line deduction, then used an estimated $15,750 standard deduction and no QBI deduction. PolicyEngine instead reached $145,669.58 of taxable income after $17,752.06 of taxable-income deductions." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model computed approximately $26,045 and then replaced it with $20,200 through an unsupported 'reconsideration' that names no valid tax adjustment or credit. It also applied a miscellaneous-itemized deduction for unreimbursed employee expenses while omitting the traced $1,264.59 QBI deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied an obsolete $6,800 standard deduction, a $4,850 personal exemption, deductible unreimbursed employee expenses, and pre-TCJA rates reaching 28%. The traced 2026 computation instead uses $17,752.06 of taxable-income deductions, including the QBI deduction, and produces $17,764.88 of ordinary tax under the applicable brackets." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted unreimbursed employee business expenses and a personal exemption while using $169,943 of AGI. It missed PolicyEngine's actual $163,421.64 AGI and $1,264.59 QBI deduction, so both its taxable-income base and ordinary tax were wrong." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The explanation omits the $6,323 estate-income treatment, the $198.35 above-the-line deduction, and the $1,264.59 QBI deduction needed to reproduce the traced taxable income. Its $19,688 answer is therefore based on an incomplete income-and-deduction shortcut rather than the ordinary-plus-preferential computation." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted employer health premiums from wages and applied sunset-style itemized deductions and personal exemptions, none of which matches the traced calculation. It also failed to apply the traced $1,264.59 QBI deduction within the $17,752.06 taxable-income deductions." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used $169,943 of AGI, invented state-income-tax and miscellaneous employee-expense deductions, and claimed a personal exemption. It missed the traced AGI treatment and $1,264.59 QBI deduction, leaving the wrong ordinary taxable-income base." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Although it reached an AGI near the traced value by subtracting employer health premiums, that is not the traced AGI pathway. Its unexplained deductions failed to reproduce $145,669.58 of taxable income and the resulting $17,764.88 ordinary tax." +us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included the $6,323 estate-income input in AGI and used only a guessed $15,400 standard deduction. It omitted the $198.35 above-the-line deduction and the $1,264.59 QBI deduction, overstating ordinary taxable income and tax." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model explicitly stated that federal income tax remains positive but submitted zero. The traced ordinary tax of $17,764.88 plus preferential tax of $6,132.56 yields $23,897.44, with no offset that reduces it to zero." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model gave only a generic estimate and did not derive the traced $163,421.64 AGI, $17,752.06 deductions, or the split between ordinary and preferential income. Its $16,811 is below the traced ordinary tax alone and therefore omits part or all of the $6,132.56 capital-gains tax." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used $169,943 of AGI and only a $16,100 standard deduction. It therefore included income excluded from the traced AGI construction and omitted the $198.35 above-the-line deduction and $1,264.59 QBI deduction, inflating ordinary taxable income." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model focused on a $387 nonitemizer charitable deduction but failed to reproduce the traced $17,752.06 total deduction, especially the $1,264.59 QBI deduction. It also did not derive the $163,421.64 AGI used by PolicyEngine." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used $169,943 of AGI and a $16,100 standard deduction rather than the traced $163,421.64 AGI and $17,752.06 taxable-income deductions. Omitting the $1,264.59 QBI deduction and the traced AGI adjustment overstated ordinary taxable income." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $169,943 AGI and $16,100 deduction do not match the traced $163,421.64 AGI and $17,752.06 deduction total. The missing $198.35 adjustment and $1,264.59 QBI deduction caused excessive ordinary-rate tax." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model relied on unspecified itemized medical deductions and unspecified nonrefundable credits instead of deriving the actual taxable-income base. The trace produces $145,669.58 of taxable income after the specific $17,752.06 deduction total, including $1,264.59 of QBI deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied restored pre-TCJA brackets, a personal exemption, and a miscellaneous deduction for employee expenses. PolicyEngine applied the operative 2026 ordinary brackets to its traced taxable income and included a $1,264.59 QBI deduction instead, producing $17,764.88 of ordinary tax." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used post-sunset 10%, 15%, and 25% brackets, a personal exemption, and miscellaneous employee-expense deductions. It also substituted a guessed $198 IRA deduction and missed the traced $1,264.59 QBI deduction, so neither taxable income nor ordinary tax followed the reference computation." +us,scenario_110,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model invented an Ohio income-tax itemized deduction and deducted unreimbursed employee expenses after a 2% floor, while retaining $169,943 of AGI. It missed the traced $163,421.64 AGI and $1,264.59 QBI deduction, producing the wrong ordinary taxable income." +us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model invented an $8,425.50 overtime deduction from the straight-time and weekly-hours facts, even though gross wages already represent the full annual amount and no qualifying overtime premium was supplied. It also used a $387 nonitemizer charitable deduction and only $297.87 of QBI-related reduction instead of the traced $1,264.59 QBI deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model asserted that PolicyEngine returned zero without performing or explaining any tax computation. The traced $17,764.88 ordinary tax and $6,132.56 preferential tax remain positive and sum to $23,897.44." +us,scenario_110,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's written computation totals $26,973, but it submitted $43,879.20, an unsupported arithmetic or transcription error. Its calculation also failed to subtract the traditional 401(k) contribution correctly and omitted the traced $198.35 adjustment and $1,264.59 QBI deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $113,617 despite the traced taxable wages, interest, dividends, and capital gains alone supporting $163,421.64 after the above-the-line deduction. It also used only a $297.87 QBI deduction and invented a $194.77 nonrefundable credit despite stating that no qualifying credit existed." us,scenario_110,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_110,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $6,889 employer-sponsored insurance premium from FICA wages without any fact establishing a pre-tax cafeteria-plan exclusion. Applying 6.2% Social Security and 1.45% Medicare to the full $100,000 of wages yields $7,650." -us,scenario_110,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly treated the $6,889 employer-sponsored insurance premium as excluded from FICA wages. The payroll tax base is the full $100,000 of wages, producing $6,200 of Social Security tax and $1,450 of Medicare tax." -us,scenario_110,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model reduced FICA wages to $93,111 by subtracting employer-sponsored insurance premiums despite no input establishing that those premiums were paid through a pre-tax FICA-exempt arrangement. FICA applies to the full $100,000, totaling $7,650." -us,scenario_110,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $6,889 employer-sponsored insurance premium from the payroll tax base. Social Security and Medicare taxes apply to the stated $100,000 of wages, yielding $7,650." -us,scenario_110,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model stated the correct 6.2% Social Security and 1.45% Medicare rules but did not apply them correctly to the $100,000 wage amount. Those components are $6,200 and $1,450, so its $8,210 answer contains a $560 computation error." -us,scenario_110,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $4,245 traditional 401(k) deferral from FICA wages. Traditional 401(k) contributions reduce federal income-tax wages but remain subject to Social Security and Medicare taxes, leaving a $100,000 payroll tax base and $7,650 of tax." -us,scenario_110,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so the required output was missing." -us,scenario_110,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model returned zero despite the head having $100,000 of employee wages subject to FICA. Those wages generate $6,200 of employee Social Security tax and $1,450 of employee Medicare tax, totaling $7,650." -us,scenario_110,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's own explanation correctly computed $6,200 of Social Security tax plus $1,450 of Medicare tax, but it submitted $9,351.90 instead of their $7,650 sum. This is a final-value transcription error that directly contradicts its stated calculation." +us,scenario_110,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $6,889 employer-sponsored insurance premium from wages even though the benchmark input does not establish that this amount is an employee pre-tax salary-reduction contribution. Applying Social Security and Medicare taxes to the full $100,000 wage amount yields $7,650." +us,scenario_110,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly treated the $6,889 employer-sponsored insurance premium as a pre-tax employee deduction from FICA wages. The payroll-tax base remains the full $100,000 of wages, producing $6,200 of Social Security tax and $1,450 of Medicare tax." +us,scenario_110,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model reduced FICA wages to $93,111 by subtracting an employer-sponsored insurance premium that is not specified as an employee pre-tax salary reduction. FICA applies to the full $100,000 wage input, yielding $7,650." +us,scenario_110,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly excluded the $6,889 employer-sponsored insurance premium from FICA wages. With the full $100,000 wage base, Social Security tax is $6,200 and Medicare tax is $1,450." +us,scenario_110,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct wage-only Social Security and Medicare rules but submitted $8,210 instead of carrying out their arithmetic. Applying 6.2% and 1.45% to $100,000 gives $6,200 plus $1,450, or $7,650." +us,scenario_110,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $4,245 traditional 401(k) deferral from FICA wages. Traditional 401(k) deferrals remain subject to Social Security and Medicare taxes, so the payroll-tax base is $100,000 and the liability is $7,650." +us,scenario_110,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model incorrectly assumed that the $6,889 employer-sponsored insurance premium was a pre-tax employee contribution excluded from FICA wages. The stated facts leave the full $100,000 subject to employee Social Security and Medicare taxes, totaling $7,650." +us,scenario_110,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, violating the required output contract." +us,scenario_110,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model returned zero and skipped employee FICA on the explicitly listed $100,000 of wages. Those wages generate $6,200 of Social Security tax and $1,450 of Medicare tax, totaling $7,650." +us,scenario_110,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's explanation correctly derived $6,200 of Social Security tax plus $1,450 of Medicare tax but submitted $9,351.90 instead of that sum. Its numeric output contradicts its own $7,650 calculation." +us,scenario_110,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model incorrectly applied Additional Medicare Tax by treating nonwage household income as though it pushed Medicare wages above the $200,000 threshold. Only the $100,000 of wages counts toward that threshold here, so Additional Medicare Tax is zero and total payroll tax is $7,650." us,scenario_110,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's stated calculation produced about $4,266 but it submitted $5,052 without any computational step supporting the increase. It also started from $169,745 rather than the traced Ohio AGI of $163,421.64." -us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted a federal-style $15,000 standard deduction in computing Ohio taxable income. Ohio taxable income here is $163,421.64 minus the $1,900 Ohio personal exemption, or $161,521.64." -us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly estimated a bracket calculation above $4,300 and then replaced it with $2,900 through an unexplained 'different structure' adjustment. No such deduction or computation step supports that reduction." -us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied a 3.5% marginal rate to the entire amount above $26,050 and then submitted $4,517 even though its own arithmetic produced about $5,325. The applicable 2026 schedule applied to $161,521.64 yields $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model started from an overstated reconstructed AGI near $169,700 and then subtracted tax-exempt interest even though that interest was already excluded from federal AGI. The traced Ohio AGI is $163,421.64 and taxable income after the exemption is $161,521.64." -us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented an Ohio itemized deduction of $10,915 for mortgage interest, property tax, and charitable gifts. This calculation subtracts only the $1,900 Ohio personal exemption from Ohio AGI and does not use those federal itemized deductions." -us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a fabricated graduated schedule including 3.5% and roughly 3.99% brackets and invoked unspecified retirement credits despite no retirement income. It also added tax-exempt interest and began from the wrong AGI rather than $163,421.64." -us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model taxed its entire income base at 2.75%, denied the applicable $1,900 personal exemption, and added $954 of tax-exempt interest without establishing an Ohio add-back. The correct base is $161,521.64 and must pass through Ohio's bracket schedule rather than a flat tax on all income." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used Ohio AGI of $169,943 and a $2,500 exemption instead of $163,421.64 and $1,900. Those errors overstated taxable income relative to the correct $161,521.64 base." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $2,981 estimate does not reflect the traced calculation: $163,421.64 of Ohio AGI less a $1,900 exemption, followed by the Ohio brackets. Its reference to unspecified 'standard deductions' indicates a deduction that is not part of this Ohio computation." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated $163,054 of federal AGI as the operative Ohio base and omitted the $1,900 personal-exemption step from its explanation. Ohio AGI is $163,421.64 and taxable income is $161,521.64." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model started from an overstated AGI of $169,943 and never specified the exemption amount or bracket arithmetic producing $3,850. The correct taxable-income base is $161,521.64." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model supplied no taxable-income or bracket derivation for $4,240.52. Applying the Ohio schedule to the traced $161,521.64 taxable income produces $4,057.47, so its answer embodies a different rate schedule or base." -us,scenario_110,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model overstated Ohio AGI as $169,943 and incorrectly treated the Ohio personal exemption as fully phased out. The applicable exemption is $1,900, reducing the traced $163,421.64 Ohio AGI to $161,521.64." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model acknowledged that Ohio tax was positive but nevertheless submitted zero. The required computation produces $4,057.47 from $161,521.64 of Ohio taxable income." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated mortgage interest and charitable contributions as relevant Ohio itemized deductions and gave no coherent tax-base or rate calculation. This Ohio computation subtracts the $1,900 personal exemption from $163,421.64 of Ohio AGI and does not deduct those expenses." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Although the model used the correct $1,900 exemption, it overstated Ohio AGI as $169,943 rather than $163,421.64. It therefore taxed $168,043 instead of the correct $161,521.64." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model gave no figures for Ohio AGI, the exemption, or the rate schedule supporting $4,418. The traced base is $161,521.64, and applying the 2026 Ohio brackets yields $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model reduced all income above the zero-tax threshold to a single 2.75% calculation. The correct 2026 Ohio bracket computation on $161,521.64 yields $4,057.47 rather than $3,905." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model overstated Ohio AGI as $169,943 and used a $1,850 exemption rather than $163,421.64 and $1,900. It also reduced the tax schedule to 2.75% above $26,050." -us,scenario_110,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model supplied neither the Ohio tax base nor any credit calculation supporting its asserted $3,200. The explicit derivation produces $4,057.47 before refundable credits, with no 'modest credits' identified to reduce it to $3,200." -us,scenario_110,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used Ohio AGI near $169,943 instead of $163,421.64. Even with an approximately correct exemption, that overstated the taxable base by more than $6,500 and produced excessive tax." -us,scenario_110,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used $169,745 of Ohio AGI and a $2,400 exemption instead of $163,421.64 and $1,900. It also applied a single 2.75% rate to all income above $26,050 rather than the applicable bracket computation." -us,scenario_110,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. -us,scenario_110,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model overstated Ohio AGI as $169,745 rather than $163,421.64, making its taxable-income base $6,323.36 too high. It then applied a single 2.75% rate and an inapplicable $20 personal-exemption credit instead of the traced bracket calculation." -us,scenario_110,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model asserted that the Ohio calculation returned zero despite the household's positive Ohio taxable income. The calculation produces $4,057.47 before refundable credits." -us,scenario_110,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model invented a $250 personal-exemption credit and an additional unspecified graduated income-tax credit, reducing its own $4,420.50 bracket result to $3,148.93. It also included tax-exempt interest and used obsolete estimated rates instead of taxing the traced $161,521.64 base under the 2026 schedule." +us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own bracket calculation produced about $4,266, but it submitted $5,052 without any computation supporting that change. It also began from an inflated $169,745 AGI instead of the traced Ohio AGI of $163,421.64." +us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted a $15,000 federal standard deduction when calculating Ohio taxable income. Ohio taxable income here is $163,421.64 minus the $1,900 Ohio personal exemption, or $161,521.64." +us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model calculated roughly $4,328 and then replaced it with $2,900 based on unspecified “federal items and exemption.” No such adjustment exists in its reasoning; the traced taxable income is $161,521.64 and yields $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied a 3.5% rate to all taxable income above $26,050, producing an unsupported bracket tax, and then submitted $4,517 despite calculating about $5,305 after its stated credit. The 2026 Ohio schedule applied to $161,521.64 produces $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model started from an inflated AGI near $169,700 and incorrectly subtracted tax-exempt interest from an AGI that already excludes it. Ohio AGI is $163,421.64, and only the $1,900 personal exemption reduces it to the relevant taxable income." +us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented an Ohio choice between a standard deduction and itemized deductions and subtracted $10,915 of federal-style itemized expenses. Ohio instead subtracts the $1,900 personal exemption from $163,421.64, and the model also used the wrong 3.5% flat-rate treatment." +us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used obsolete graduated brackets, including 3.5% and 3.99% rates, and invoked retirement and other credits despite no qualifying facts. It also inflated Ohio AGI by adding tax-exempt interest and other unsupported amounts rather than using $163,421.64." +us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model added $954 of tax-exempt interest to an incorrect $163,054 AGI and asserted that Ohio had no personal exemption. The traced calculation uses $163,421.64 of Ohio AGI and subtracts the $1,900 exemption before applying the tax schedule." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used an inflated Ohio AGI of $169,943 and a $2,500 exemption. The applicable figures are $163,421.64 of Ohio AGI and a $1,900 exemption, giving $161,521.64 of taxable income." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies an unspecified standard deduction that is not part of this Ohio derivation. Ohio taxable income is $161,521.64 after subtracting only the $1,900 personal exemption from Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used $163,054 as federal AGI without deriving the traced Ohio AGI of $163,421.64 or subtracting the $1,900 personal exemption correctly. Applying the Ohio schedule to $161,521.64 yields $4,057.47, not $4,241." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model started from an inflated $169,943 AGI and applied unspecified exemptions and progressive brackets. The correct taxable base is $161,521.64 after the $1,900 Ohio personal exemption." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model supplied no calculation supporting $4,240.52. The traced steps are $163,421.64 of Ohio AGI less the $1,900 exemption, followed by the Ohio schedule, which yields $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model overstated Ohio AGI as $169,943 and incorrectly treated the Ohio personal exemption as fully phased out. The applicable exemption is $1,900, reducing the traced $163,421.64 AGI to $161,521.64." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model acknowledged that Ohio tax was positive but submitted zero instead of performing the requested estimate. The provided facts produce $161,521.64 of Ohio taxable income and $4,057.47 of tax." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated mortgage interest and charitable contributions as Ohio deductions and produced an unsupported approximation. This Ohio calculation subtracts the $1,900 personal exemption from $163,421.64 and does not use those itemized expenses." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model correctly identified the $1,900 exemption but applied it to an inflated $169,943 AGI. Ohio AGI is $163,421.64, so taxable income is $161,521.64 rather than $168,043." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The model asserted that it applied the exemption and rate schedule but provided no derivation for $4,418. Applying those steps to the traced $163,421.64 Ohio AGI produces $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's $3,905 result reflects applying its 2.75% shortcut to the wrong taxable base. The applicable base is $161,521.64 after subtracting the $1,900 exemption from $163,421.64." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used an inflated $169,943 AGI and the wrong $1,850 personal exemption. The traced values are $163,421.64 of Ohio AGI and a $1,900 exemption." +us,scenario_110,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model supplied only a generic assertion about AGI adjustments and modest credits, with no computation supporting $3,200. The traced taxable income of $161,521.64 produces $4,057.47 before refundable credits." +us,scenario_110,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model applied the $1,900 exemption to an inflated $169,943 AGI and then used an obsolete 2.75%/3.5% bracket structure. The correct taxable income is $161,521.64." +us,scenario_110,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an inflated $169,745 AGI and an incorrect $2,400 exemption. Ohio AGI is $163,421.64 and the applicable personal exemption is $1,900." +us,scenario_110,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model started from an inflated AGI near $169,943 and applied a 3.5% upper bracket to taxable income near $167,500. The 2026 Ohio schedule must instead be applied to $161,521.64." +us,scenario_110,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. +us,scenario_110,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used $169,745 rather than the traced $163,421.64 Ohio AGI and then subtracted an inapplicable $20 personal-exemption credit. The $1,900 exemption is a deduction in the traced calculation, yielding taxable income of $161,521.64." +us,scenario_110,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model falsely asserted that the Ohio calculation returned zero and provided no tax computation. The household has $161,521.64 of Ohio taxable income, which produces $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model included tax-exempt interest in its starting income, used estimated obsolete bracket rates and thresholds, invented a $250 personal-exemption credit, and then applied an unexplained further reduction. The traced calculation instead subtracts a $1,900 exemption from $163,421.64 and directly yields $4,057.47 under the 2026 schedule." +us,scenario_110,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model reduced Ohio taxable income to $96,517 using an unspecified standard deduction and exemptions. The actual taxable income is $161,521.64 because the traced calculation subtracts only the $1,900 Ohio personal exemption from $163,421.64." us,scenario_110,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_111,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"claude-sonnet-5 misread the Social Security taxation thresholds: provisional income is $18,240 plus half of $25,400, or $30,940, which is below the married-joint base threshold rather than above the $44,000 higher threshold. It therefore invented roughly $16,600 of taxable Social Security and then submitted a positive tax even after its own standard-deduction calculation produced $0 taxable income." -us,scenario_111,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"qwen-3.7-max incorrectly included $5,405.50 of Social Security in AGI even though provisional income is below the joint-filer Social Security taxation threshold, so AGI is only the $18,240 taxable IRA distribution. Its own reasoning then reached $0 taxable income and $0 tax before refundable credits, but it submitted $1,400 anyway, a phantom liability unsupported by the taxable-income calculation." -us,scenario_111,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly applied an aged/disabled Medicaid pathway and assumed SSI-style income disregards reduced $43,640 of Social Security and IRA income below Washington's senior Medicaid limits. PolicyEngine found no Medicaid category for the head: MAGI is 2.02 x FPL for MAGI categories, SSI received is zero, and age/disability/assets alone do not establish eligibility." -us,scenario_111,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model conflated Washington Medicaid expansion MAGI rules with aged/blind/disabled pathways and treated low assets plus disability as enough for eligibility. The head is age 72 with MAGI at 2.02 x FPL and no SSI receipt, so the modeled Medicaid category is NONE rather than expansion or SSI-related eligibility." -us,scenario_111,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented an SSI-related aged/blind/disabled Medicaid pathway using resource limits, income disregards, and spend-down medical deductions. In the engine trace the head receives $0 SSI and qualifies through no Medicaid category, so blindness/disability, low bank assets, and medical expenses do not convert $43,640 of income into Medicaid eligibility." -us,scenario_111,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model first identified that no wages were provided and that wage-based employee payroll tax is $0, then overrode that result by taxing retirement income as a mandatory payroll-tax base. Social Security retirement income and taxable IRA distributions are not employee-side payroll-taxable earnings, so the correct payroll tax remains $0." -us,scenario_111,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly treated Washington broad-based categorical eligibility at 200% of poverty as making the household SNAP-eligible despite its income-based disqualification under the applicable PolicyEngine SNAP eligibility test. It then applied the $24 monthly minimum allotment to an ineligible household, but the minimum allotment is only available after eligibility is established." -us,scenario_111,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated blindness and disability as automatic Medicaid eligibility in Washington and explicitly ignored income limits for disability-related pathways. PolicyEngine requires an actual eligibility category and income qualification; this spouse has no Medicaid category and MAGI of 2.02 FPL, above the adult expansion limit." -us,scenario_111,spouse_medicaid_eligible,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model correctly noticed household income exceeds the ACA expansion limit, then incorrectly switched to a disability pathway using only the spouse's own income. PolicyEngine counts the relevant household income for this case and finds no disability-based Medicaid category, so the spouse's MAGI is 2.02 FPL and eligibility is false." -us,scenario_111,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model assumed that being blind and disabled was enough to qualify through Washington ABD Medicaid without applying the pathway's income and eligibility tests. PolicyEngine found the spouse qualifies through none of the Medicaid pathways, and the MAGI adult pathway fails because household income is 2.02 FPL." -us,scenario_111,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model acknowledged the household exceeds the 138% FPL ACA expansion limit but then treated blindness and disability as opening an income-independent Medicaid pathway. PolicyEngine applies category-specific tests and found no SSI-related, disability-based, or other Medicaid category for the spouse." -us,scenario_111,spouse_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,The model incorrectly concluded that countable household income was low enough for Washington blind/disabled Medicaid after considering assets. The spouse's relevant income position is above the available Medicaid limits: PolicyEngine calculates MAGI at 2.02 FPL and finds no ABD or other eligibility category. -us,scenario_111,spouse_medicaid_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model relied on vague ABD exclusions and resource assumptions to conclude countable income was within Washington's SSI-related Medicaid limit. PolicyEngine found no SSI receipt, no non-MAGI disability eligibility category, and MAGI above the adult expansion threshold at 2.02 FPL." -us,scenario_111,spouse_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model computed MAGI as only the head's taxable IRA distributions and excluded Social Security retirement income from the Medicaid income comparison. PolicyEngine's Medicaid MAGI for the spouse is based on household income of about $43,640, producing 2.02 FPL, above the 138% FPL expansion limit; the ABD shortcut also fails because the engine found no disability-based eligibility category." -us,scenario_111,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated the disability flag as automatically conferring Medicare eligibility regardless of age. It failed to require an under-65 Medicare pathway such as qualifying SSDI entitlement and the applicable waiting period, neither of which was provided." -us,scenario_111,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model incorrectly converted the spouse's disability status directly into Medicare eligibility. At age 49, disability alone is insufficient without the required disability-benefit entitlement and waiting-period conditions." -us,scenario_111,spouse_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,The model acknowledged that disability can lead to Medicare through Social Security disability benefits but then assumed that pathway was satisfied. The prompt supplies no SSDI entitlement or completed waiting period and requires unlisted statuses to be false. -us,scenario_111,spouse_medicare_eligible,claude-opus-5,llm_error,age_disability,False,The model asserted a PolicyEngine rule under which disability alone qualifies an under-65 person for Medicare. The spouse is 49 and lacks the additional entitlement conditions required for disability-based Medicare. -us,scenario_111,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated inclusion of disability-based Medicare pathways as proof that this disabled spouse qualified. It omitted the required SSDI entitlement and waiting-period conditions, so the age-49 spouse remains ineligible." -us,scenario_111,spouse_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,The model incorrectly stated that PolicyEngine treats every disabled individual as Medicare-eligible regardless of age. A 49-year-old does not qualify from the disability flag alone; the necessary under-65 Medicare entitlement conditions were absent. -us,scenario_112,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model included the $13,000 financial assistance in ordinary taxable income even though that input does not enter federal AGI. Excluding it yields AGI of $4,842.82 after the $3,000 capital-loss deduction, which is fully absorbed by the standard deduction, so its claimed $1,743 of taxable income and $174.30 tax are spurious." -us,scenario_112,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly derived an EITC of about $453 from $5,923 of wages at the childless phase-in rate, then submitted $191 without applying any rule or arithmetic that produces that amount. It discarded its own correct EITC computation instead of returning $453.10." -us,scenario_112,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly treated negative taxable income and zero federal tax liability as barriers to the EITC. The EITC is refundable and phases in from $5,923 of earned wages, producing $453.10 despite the deductions and lack of children." -us,scenario_112,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly calculated approximately $453 from the 7.65% childless EITC phase-in and correctly found no phaseout, then replaced that result with an unsupported $50 estimate. Low earnings in the phase-in range generate the credit rather than reducing it to a nominal amount." -us,scenario_112,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model incorrectly treated negative net income from capital losses as eliminating the childless EITC. The credit calculation retains $5,923 of earned wages as its phase-in base, yielding $453.10, while the capital loss does not disqualify the filer under the investment-income test." -us,scenario_112,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model stated the correct phase-in calculation of roughly $453 but then substituted $492 by treating the filer as closer to the maximum-credit plateau. At $5,923 of earned wages, the applicable phase-in calculation yields $453.10, not a plateau-based estimate." -us,scenario_112,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model incorrectly included farm rent in earned income and then treated the capital loss and negative AGI as extinguishing the EITC. PolicyEngine applies the childless phase-in to the $5,923 of wages, and the negative capital result does not disqualify the filer, producing $453.10." -us,scenario_112,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the listed employer-sponsored insurance premiums from gross wages when determining EITC earned income. The supplied $5,923 gross wage amount remains the earned-income base, so the childless EITC is $453.10." -us,scenario_112,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a 7.67% estimated phase-in rate instead of the exact 2026 EITC parameters and calculation. Applying the engine's childless EITC computation to $5,923 of wages yields $453.10 rather than $454.29." -us,scenario_112,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly asserted that the filer failed the EITC income or eligibility requirements. A 31-year-old childless single filer with $5,923 of earned wages is within the eligible phase-in range and receives $453.10." -us,scenario_112,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the employer-sponsored insurance premium as a pretax deduction that fully offsets the separately reported gross wages for EITC purposes. The $5,923 wage input supplies positive earned income and generates a $453.10 childless EITC." -us,scenario_112,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted $8,389 of employer-sponsored insurance premiums from $5,923 of gross wages and set EITC earned income to zero. The wages remain the EITC phase-in base, producing $453.10." -us,scenario_112,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model incorrectly based the phase-in credit on the lesser of earned income and AGI, using $4,843 instead of $5,923. In this case the capital-loss-reduced AGI does not replace wages as the EITC phase-in base, so the calculation yields $453.10." -us,scenario_112,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the absence of qualifying children as eliminating every refundable federal credit and omitted the childless EITC pathway. The age-31 filer qualifies on $5,923 of wages and receives $453.10." -us,scenario_112,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a child or an explicitly listed credit input before recognizing a refundable credit. EITC eligibility is computed from the stated age, filing status, and $5,923 of earned wages, which produce $453.10." -us,scenario_112,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly declared $5,923 insufficient earned income for the childless EITC. The EITC phases in from the first dollars of earnings for this eligible 31-year-old filer and reaches $453.10 at the stated wages." -us,scenario_112,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model conflated an ACA-style premium subtraction with EITC earned income and assumed very low earnings make the childless EITC zero. The EITC instead phases in on the full $5,923 wage amount, yielding $453.10." -us,scenario_112,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model correctly computed about $453 from wages but then imposed an unsupported reduction for the long-term capital loss. A capital loss does not reduce a positive phase-in EITC after the filer passes the investment-income test, so the credit remains $453.10." -us,scenario_112,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly noted that Texas did not expand Medicaid but then applied a generic low-income adult pathway that Texas does not provide for this 31-year-old adult. It treated MAGI at 0.30 FPL as independently qualifying and never applied the required categorical screen, under which the head has medicaid_category = NONE." -us,scenario_112,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model reduced Texas Medicaid eligibility to low income and minimal assets. It missed that this head is not in any active Texas Medicaid eligibility category, so passing an income screen does not matter without a qualifying categorical pathway." -us,scenario_112,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model applied a non-expansion-state rule as if low income alone qualifies a single adult for Medicaid in Texas. The correct computation rejects eligibility because the head is age 31, non-dependent, receives no SSI, and qualifies through none of the Medicaid categories." -us,scenario_112,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model explicitly assumed Texas has expanded adult Medicaid coverage. Texas is not treated as providing an expansion adult pathway here, and the head's low wages do not overcome medicaid_category = NONE." -us,scenario_112,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly identified that no qualifying WIC category was listed for the 31-year-old head, then contradicted that rule by awarding eligibility based on low income alone. It treated the 185% FPL income test as sufficient for WIC and failed to enforce the categorical demographic requirement." -us,scenario_112,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model incorrectly treated the listed employer-sponsored insurance premiums as a pre-tax cafeteria-plan deduction from FICA wages. PolicyEngine applied employee Social Security and Medicare tax to the $5,923 of gross wages, producing $367.22 plus $85.88 rather than zero." -us,scenario_112,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model zeroed out FICA wages by subtracting employer-sponsored health insurance premiums from gross wages. The payroll tax base remained the $5,923 of wages, so employee Social Security and Medicare taxes still applied." -us,scenario_112,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model misapplied the health-premium input as a full offset against FICA-taxable wages. PolicyEngine did not subtract that premium from wages for payroll tax, so the head owed employee Social Security tax and Medicare tax on $5,923." -us,scenario_112,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model used the right tax base and rates but miscomputed the total. Applying 6.2% Social Security and 1.45% Medicare to $5,923 yields $367.22 plus $85.88, not $455.00." -us,scenario_112,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct payroll-tax components, $367.21 of Social Security tax and $85.88 of Medicare tax, but then added them to an impossible $479.38 total. The correct summed payroll tax is $453.10 after PolicyEngine's component calculation." -us,scenario_112,self_employment_tax,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated the $1,920 farm rent line as positive net self-employment income by itself and ignored the engine's net loss from self-employment activity. It then mechanically applied the 92.35% net earnings factor and 15.3% SECA rate to a positive base that PolicyEngine reduced below zero, creating tax where no positive net earnings existed." -us,scenario_112,self_employment_tax,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model stated the rental income was passive but then submitted the alternative calculation that treats the $1,920 as active net self-employment income. It failed to carry through the rule that self-employment tax uses positive net self-employment earnings after losses, which PolicyEngine computed as a loss rather than a taxable SE base." -us,scenario_112,self_employment_tax,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly combined farm rent and financial assistance into a positive self-employment tax base and described offsetting negative capital gains, a step that does not match PolicyEngine's self-employment earnings computation. The correct derivation produces a net self-employment loss, and SECA tax is not assessed on a loss." -us,scenario_112,self_employment_tax,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the $1,920 farm rent amount as the entire net self-employment base and applied the 92.35% factor and 15.3% SECA rate. It missed that PolicyEngine's self-employment computation nets to an approximately -$2,101 loss, leaving no positive net earnings subject to self-employment tax." -us,scenario_112,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $1,920 of farm rent income in SNAP gross and net income, producing monthly figures of about $1,737 and $1,429. Excluding that item yields gross income of $1,576.90 and net income of $1,269.19, so both tests are passed and the minimum allotment applies." -us,scenario_112,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model counted the $1,920 farm rent amount in SNAP gross income and therefore overstated monthly gross income as $1,737. SNAP gross income is $1,576.90, which passes the gross-income test." -us,scenario_112,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly added farm rent income to countable SNAP income, causing its asserted failure of the gross-income test. The applicable gross income is $1,576.90 per month, and the household qualifies for the minimum allotment." -us,scenario_112,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded the $13,000 financial-assistance amount from SNAP income while including farm rent income, radically understating net income and calculating a near-maximum benefit. The financial assistance is counted, the farm rent amount is not, and net income of $1,269.19 makes the expected contribution exceed the maximum allotment, leaving only the minimum allotment." -us,scenario_112,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $13,000 financial assistance from countable SNAP income and treated the household as having only about $654 of monthly income. Countable net income is $1,269.19 per month, so the benefit is the minimum allotment rather than roughly $275 per month." -us,scenario_112,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $1,920 of farm rent income in the SNAP gross-income test. Removing that amount gives monthly gross income of $1,576.90, below the applicable limit, and the eligible household receives the minimum allotment." -us,scenario_112,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model treated farm rent income as countable SNAP income and consequently overstated gross income above the limit. SNAP gross income is $1,576.90 per month, and the household passes both income tests." -us,scenario_112,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model’s $1,737 monthly gross-income figure includes the $1,920 farm rent amount. The SNAP calculation excludes that item, producing $1,576.90 of monthly gross income and eligibility for the minimum allotment." -us,scenario_112,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model counted farm rent income and obtained net income of $1,440.20. The correct deduction sequence starts from $1,576.90 of gross income and yields $1,269.19 of net income, which is below the net-income limit." -us,scenario_112,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The answer incorrectly implies that the household fails SNAP eligibility without performing the stated tests. The household passes the gross, net, asset, work, and immigration tests and also has categorical eligibility through TANF non-cash assistance, requiring the annual minimum allotment of $287.68." -us,scenario_112,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly converted an expected contribution above the maximum allotment into a zero benefit. Once the household is eligible, this circumstance triggers the SNAP minimum allotment, totaling $287.68 for the year." -us,scenario_112,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted farm rent income, overstating SNAP net income as about $1,440 instead of $1,269.19. It also treated a contribution above the maximum allotment as eliminating benefits, when the eligible household receives the minimum allotment." -us,scenario_112,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model treated the excess of the expected contribution over the maximum allotment as producing no SNAP payment. An eligible one-person household instead receives the minimum allotment, which totals $287.68 annually." -us,scenario_112,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included $1,920 of farm rent income in countable SNAP income and therefore calculated net income of about $1,434. Excluding that item yields $1,269.19 of net income, below the $1,304.17 limit." -us,scenario_112,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,The answer’s unsupported resource-and-income shortcut misses that bank assets are only $10 and the household passes both income tests. It also overlooks categorical eligibility through TANF non-cash assistance and the minimum allotment payable to an eligible household. -us,scenario_112,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly claimed that the prompt lacked the inputs needed to determine SNAP eligibility even though it specified a one-person household, income, assets, and all unlisted facts as zero or false. Those inputs establish categorical eligibility through TANF non-cash assistance, successful eligibility tests, and a $287.68 minimum annual allotment." -us,scenario_112,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model’s $20,843 countable-income total incorrectly includes $1,920 of farm rent income. SNAP gross income is $1,576.90 per month, so the household does not fail the gross-income limit." -us,scenario_112,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the farm rent amount alongside wages and financial assistance for the SNAP gross-income test. Without the farm rent item, gross income is $1,576.90 per month and the household qualifies." -us,scenario_112,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model included farm rent income before applying the earned-income and standard deductions, causing a false net-income-test failure. The proper calculation yields $1,269.19 of monthly net income, below the $1,304.17 limit." -us,scenario_112,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated farm rent income as part of SNAP gross income. The countable gross amount is $1,576.90 per month, which passes the gross-income test." -us,scenario_112,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model asserted generic income, asset, or trigger failures without applying the household’s facts. Assets are only $10 for the relevant test, income passes both limits, and TANF non-cash assistance supplies categorical eligibility." -us,scenario_112,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model’s stated gross-or-net failure results from treating the farm rent amount as countable SNAP income. Gross income is $1,576.90 and net income is $1,269.19 per month, so neither test disqualifies the household." -us,scenario_112,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model’s $20,843 gross-income calculation incorrectly includes the $1,920 farm rent amount. Removing it produces countable gross income below the limit and preserves eligibility for the minimum allotment." -us,scenario_112,snap,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly included farm rent income in its $20,843 countable-income total. The applicable SNAP gross income is $1,576.90 per month, which is below the gross-income threshold." -us,scenario_112,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model expressly denied TANF categorical eligibility even though PolicyEngine assigns categorical eligibility through TANF non-cash assistance. It also treated the expected contribution as eliminating the allotment, whereas an eligible household receives the minimum benefit." -us,scenario_112,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly treated a one-person household with wage income as categorically ineligible without applying any SNAP thresholds or deductions. This household passes all eligibility tests, including categorical eligibility through TANF non-cash assistance, and receives the minimum allotment." -us,scenario_112,snap,qwen-3.7-max,llm_error,other,False,"The model’s submitted $3,576 contradicts its own calculation that the ordinary allotment formula produced zero and then substitutes an unsupported near-maximum estimate. The correct calculation counts the financial assistance, excludes the farm rent amount, derives $1,269.19 of monthly net income, and applies the minimum allotment for an annual benefit of $287.68." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted computed Virginia income tax from SALT, used raw charitable gifts and an incorrect medical deduction, and therefore counted only $19,518 of itemized deductions instead of $24,787.23. It then abandoned its own $11,889 bracket calculation and submitted $8,971 without a valid tax computation." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model excluded taxable Social Security from gross income, treated the QBI deduction as an AGI adjustment without actually subtracting it, and used a fabricated $28,300 standard deduction. It also added employee Medicare tax and an invented withholding adjustment to an output that contains federal individual income tax only." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used a $10,000 SALT deduction despite identifying only $4,064 of property tax, omitted OTC medical expenses, and initially applied the full $6,000 senior deduction instead of its phased-down $4,105.65 amount. Its final $5,489 does not follow from its own recomputed tax of about $10,711." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented $4,500 of mortgage interest from the mortgage balance and omitted the $4,105.65 additional senior deduction. The trace instead uses $24,787.23 of itemized deductions, the senior deduction, and the QBI deduction to reach $72,807.33 of taxable income." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated partnership/S-corporation income as self-employment income, deducted half of an invented self-employment tax, and selected the standard deduction rather than the larger $24,787.23 itemized amount. It then reduced its own roughly $11,900 bracket result to $6,900 without any applicable nonrefundable credit." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model inferred $6,693 of mortgage interest from the mortgage balance even though unlisted expenses are zero, while omitting computed Virginia income tax from SALT and the $4,105.65 senior deduction. Those errors produced $75,490 rather than $72,807.33 of taxable income." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model counted only property tax in SALT, used raw charitable contributions and an incorrect medical deduction, and omitted the phased senior deduction. It therefore used $19,518 rather than $24,787.23 of itemized deductions and overstated taxable income." -us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $21,069.57 and omitted the $4,105.65 additional senior deduction. The required deductions total $33,765.23 after combining $24,787.23 itemized, $4,872.35 QBI, and $4,105.65 senior deductions." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented a deductible half of self-employment tax for partnership/S-corporation income, applied a personal exemption, and used obsolete 10%/15%/25% brackets. The correct computation uses $106,572.56 AGI, no personal exemption, $72,807.33 taxable income, and the operative 2026 rates." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used $110,094 of gross income instead of the traced $106,572.56 AGI and arrived at roughly $81,300 of taxable income. Applying the full deduction stack produces $72,807.33, not $81,300." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model explicitly applied pre-TCJA personal exemptions and the associated post-sunset framework. The 2026 computation instead uses the operative rate schedule and deductions, including the QBI and phased additional senior deductions, on $72,807.33 of taxable income." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied an inapplicable $5,050 personal exemption and sunset-era tax brackets while understating itemized deductions. The operative calculation has no personal exemption and taxes $72,807.33 under the 2026 rate schedule." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $22,018.04 and omitted the $4,105.65 additional senior deduction. Those omissions raised taxable income from the traced $72,807.33 to $79,682.36." -us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model incorrectly assumed the QBI deduction had expired, restored a personal exemption, and used obsolete 10%/15%/25% brackets. The operative rules allow a $4,872.35 QBI deduction and a $4,105.65 senior deduction and apply the 2026 rate schedule to $72,807.33." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the standard and age-based deductions as sufficient to erase more than $106,000 of AGI. Even after the larger itemized deduction, QBI deduction, and senior deduction, taxable income remains $72,807.33 and generates $10,729.61 of tax." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model excluded or heavily discounted taxable retirement and Social Security income and concluded that deductions eliminated the liability. The traced AGI is $106,572.56, including $19,828.80 of taxable Social Security, and deductions leave $72,807.33 taxable." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model applied a 2026 personal exemption and post-sunset brackets while counting only about $17,518 of itemized deductions. The calculation instead uses $24,787.23 of itemized deductions plus QBI and senior deductions, with no personal exemption, under the operative 2026 rates." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted half of self-employment tax from partnership/S-corporation income and used the standard-deduction pathway, reducing taxable income to about $67,020. The correct optimal pathway itemizes and yields taxable income of $72,807.33 without that invented adjustment." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model selected the standard deduction and a non-itemizer charitable deduction instead of the larger $24,787.23 itemized deduction. Combining the correct itemized amount with the $4,872.35 QBI and $4,105.65 senior deductions yields $72,807.33, not $78,445." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The submitted $11,540 implies that the model did not apply the traced deduction amounts despite naming their categories. The itemized, QBI, and senior deductions total $33,765.23 and reduce taxable income to $72,807.33 before the rate calculation." -us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model supplied no usable derivation, and its $10,234 does not equal the tax on the traced $72,807.33 of taxable income. Applying the 2026 brackets to that amount yields $10,729.61." -us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model restored a personal exemption and obsolete 10%/15%/25% brackets and omitted both the QBI and additional senior deductions. The operative computation uses no personal exemption and applies 2026 rates after total deductions of $33,765.23." -us,scenario_114,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $22,264 and omitted the $4,105.65 additional senior deduction. The trace uses $24,787.23 of itemized deductions and reduces taxable income to $72,807.33 rather than $79,437." -us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the required structured quantity was missing." -us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model chose the $18,150 standard deduction after understating itemized deductions at $17,518 and then added a non-itemizer charitable deduction. The correct itemized amount is $24,787.23, which with the QBI and senior deductions produces $72,807.33 of taxable income." -us,scenario_114,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model acknowledged positive taxable income but then reduced tax to zero through unspecified nonrefundable credits, although no such credits apply. It also invented mortgage interest from the outstanding balance; the actual deduction stack leaves $72,807.33 taxable and $10,729.61 due." -us,scenario_114,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted $19,828.80 of taxable Social Security from income, used the standard deduction instead of itemizing, and omitted the $4,105.65 senior deduction. It also applied the brackets incorrectly, since $66,122 of taxable income would not produce its stated $15,654 under the operative schedule." -us,scenario_114,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model computed childless EITC using only $22,700 of wages and applied a wage-only phaseout calculation. It ignored the household's pension, IRA distribution, Social Security, partnership income, interest, and other income that put total income far above the childless EITC limit, so the EITC is $0 and no other refundable federal credit applies." -us,scenario_114,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable federal_refundable_credits value or supporting explanation. The correct computation evaluates EITC, refundable AOTC, refundable CTC, recovery rebate credit, and refundable payroll tax credit, and each is $0 for this 69-year-old single adult with no children and roughly $110,072 of household income." -us,scenario_114,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model stated the correct employee-side components, $1,407.40 of Social Security tax and $329.15 of Medicare tax, but submitted $3,470.30 instead of their $1,736.55 sum. Its submitted value is consistent with erroneously adding a second payroll-tax layer after correctly excluding self-employment tax and Virginia payroll tax in the explanation." -us,scenario_114,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model computed the exact correct payroll tax, $1,407.40 plus $329.15 equals $1,736.55, then replaced it with $1,481.30 under an unsupported simplification or rounding adjustment. Payroll tax is not rounded or simplified downward after applying the statutory employee Social Security and Medicare rates." -us,scenario_114,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable payroll_tax value or explanation. The required computation is employee Social Security tax of $1,407.40 plus employee Medicare tax of $329.15, yielding $1,736.55." -us,scenario_114,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model computed the correct federal employee FICA amount of $1,736.55, then added an extra Virginia state payroll tax amount to reach $1,775.70. Virginia has no mandatory employee payroll tax included in this payroll_tax output for this household, so no state payroll add-on is applied." -us,scenario_114,self_employment_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model incorrectly reclassified the $24,362 partnership or S-corp income input as net self-employment income and applied the 92.35% adjustment and 15.3% tax rate. That income does not populate the self-employment earnings base, which remains zero." -us,scenario_114,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly placed the $24,362 partnership or S-corp income in the self-employment tax base, then compounded the error by applying the 92.35% factor twice. The specified input produces no net earnings from self-employment, so no self-employment tax is assessed." -us,scenario_114,self_employment_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model treated all $24,362 of partnership or S-corp income as self-employment earnings and calculated both Social Security and Medicare self-employment tax on 92.35% of it. The self-employment earnings input is zero, so neither component applies." -us,scenario_114,self_employment_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model assumed the combined partnership or S-corp income field represented a general partner's self-employment income. No general-partner status or net self-employment earnings were supplied, and the partnership or S-corp income input does not enter the self-employment tax base." -us,scenario_114,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model inferred active partnership income from the $24,362 partnership or S-corp income even though unlisted statuses are false and unlisted self-employment income is zero. Its 92.35%-of-income calculation therefore taxes an amount outside the self-employment tax base." -us,scenario_114,self_employment_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model directly equated the $24,362 partnership income component with self-employment income and applied the standard 15.3% tax to 92.35% of it. The benchmark supplies no net earnings from self-employment, leaving the taxable base at zero." -us,scenario_114,self_employment_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model applied the 92.35% adjustment and 15.3% rate to the $24,362 partnership or S-corp income input. That input is not net earnings from self-employment, so the correct self-employment tax base is zero." -us,scenario_114,self_employment_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model wrongly treated the $24,362 partnership or S-corp income as self-employment earnings, then introduced deductions, an AGI adjustment, and a Virginia offset that do not determine gross federal self-employment tax liability. With no net self-employment earnings, the calculation stops at a zero tax base." -us,scenario_114,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model classified the $24,362 partnership or S-corp income as net self-employment income and applied Social Security and Medicare self-employment tax to 92.35% of it. The supplied income field does not populate net earnings from self-employment, which are zero." -us,scenario_114,self_employment_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model applied self-employment tax to the $24,362 partnership income without establishing any net earnings from self-employment. The benchmark's self-employment tax base is zero because no self-employment income is listed." -us,scenario_114,self_employment_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model treated the $24,362 partnership or S-corp income as self-employment earnings and applied 15.3% to 92.35% of that amount. The Social Security wage-base comparison is irrelevant because net self-employment earnings are zero." -us,scenario_114,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"The answer's reference to taxing partnership income after deductions shows that the model placed the $24,362 partnership or S-corp income in the self-employment tax base. No net self-employment income was supplied, so the base and resulting tax are both zero." -us,scenario_114,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model omitted the required `self_employment_tax` output entirely. It therefore failed the required output contract rather than completing the zero-base calculation. -us,scenario_114,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model wrongly treated the $24,362 partnership or S-corp income as self-employment income and applied the 92.35% and 15.3% factors; its submitted value also contradicts the amount produced by its own displayed formula. The specified self-employment earnings base is zero, so Additional Medicare Tax and wage-base considerations never arise." -us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own taxable-income estimate of roughly $66,296 produces about $3,555 under the brackets it stated, yet it submitted $2,597 without any supporting adjustment. It also understated the traced Virginia itemized deductions and omitted the $800 aged exemption, but those small differences do not explain its unsupported final reduction." -us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted charitable gifts and employee expenses in computing federal AGI, omitted taxable Social Security and several retirement-income effects, and then applied a fictitious $28,000 Virginia standard deduction. Virginia taxable income is $65,593.83, not an amount below the tax threshold." -us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model chose the standard deduction instead of the larger $19,419.94 Virginia itemized deduction and omitted the $1,730 exemptions, leading its reasoning to $4,227. It then submitted $1,854, a number that contradicts its own stated derivation and has no identified deduction or credit supporting it." -us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model did not complete the itemized-deduction calculation and instead anchored its estimate near the standard-deduction result. The correct Virginia itemized deduction is $19,419.94 and the exemptions total $1,730, leaving $65,593.83 taxable rather than the higher base implicit in $4,014." -us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction instead of the larger $19,419.94 itemized deduction and treated the $1,730 exemptions as tax credits. They reduce taxable income, producing $65,593.83 before the Virginia brackets are applied." -us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used the standard deduction despite $19,419.94 of allowable itemized deductions, then subtracted the $930 personal and $800 aged exemptions directly from tax as credits. Virginia exemptions reduce taxable income, not tax liability, yielding taxable income of $65,593.83 and tax of $3,514.15." -us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model wrongly treated Social Security as federally excluded and allowed a $12,000 Virginia age deduction even though the income-based age deduction is fully phased out. It also omitted the $1,730 personal and aged exemptions and understated itemized deductions; the applicable reductions produce $65,593.83 of taxable income." -us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model allowed a $12,000 age deduction that is fully eliminated by the income limitation, used the standard deduction instead of $19,419.94 of itemized deductions, and omitted $1,730 of exemptions. Those errors happened to offset substantially but do not reproduce the correct $65,593.83 taxable income." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated itemized deductions at $23,948.60 and allowed only an $800 exemption. The trace allows $19,419.94 of itemized deductions plus both the $930 personal and $800 aged exemptions, leaving taxable income of $65,593.83 rather than $60,294.40." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $4,180 implies a taxable base near $77,174, consistent with using roughly the standard deduction rather than the larger itemized deduction and full exemptions. Subtracting $19,419.94 of itemized deductions and $1,730 of exemptions from Virginia AGI yields $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The submitted $3,726 implies Virginia taxable income near $69,278, so the model did not apply the full traced combination of $19,419.94 in itemized deductions and $1,730 in exemptions. Those reductions leave $65,593.83 and produce $3,514.15 under the Virginia brackets." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions as $15,511 and misstated exemptions as $1,600. The correct amounts are $19,419.94 and $1,730, which reduce taxable income from its $69,633 estimate to $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly applied $1,730 of exemptions but understated Virginia itemized deductions by $1,901.90, using $17,518.04 instead of $19,419.94. That exact omission overstated taxable income by $1,901.90 and tax by about $109.36." -us,scenario_114,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model described a taxable-income calculation governed by Virginia's 5.75% top rate but submitted $14,036.62, which cannot result from the income and brackets it stated. It also invented a pension deduction and used the standard deduction instead of the traced itemized deductions, but neither error supports a tax remotely near its answer." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated more than $86,000 of Virginia AGI as fully sheltered by deductions and exemptions. The allowable $19,419.94 itemized deduction and $1,730 exemptions leave $65,593.83 taxable, so liability is not zero." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated retirement-income exclusions and deductions as sufficient to erase Virginia taxable income. After the Social Security subtraction, itemized deductions, and exemptions, $65,593.83 remains taxable and generates $3,514.15." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model understated Virginia itemized deductions by $1,901.90, using $17,518 instead of $19,419.94. With the correctly stated $1,730 exemptions, the full deduction reduces taxable income to $65,593.83 rather than about $67,496." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model applied an age deduction even though the income-limited $12,000 age deduction is fully phased out, driving taxable income down to about $51,647. The applicable age-related reduction is the $800 aged exemption within total exemptions of $1,730, leaving $65,593.83 taxable." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used an $8,750 standard deduction instead of the larger $19,419.94 itemized deduction and omitted the $800 aged exemption. Applying the full $1,730 exemptions with itemization reduces taxable income from its roughly $77,064 estimate to $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The submitted $4,658 implies taxable income near $85,487, meaning the model effectively failed to apply most of the traced deductions and exemptions after the Social Security subtraction. Virginia taxable income is $65,593.83 after $19,419.94 of itemized deductions and $1,730 of exemptions." -us,scenario_114,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the standard deduction and exemptions eliminate Virginia taxable income. Even with the larger $19,419.94 itemized deduction and $1,730 exemptions, $65,593.83 remains taxable." -us,scenario_114,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions by $1,901.94, using $17,518 instead of $19,419.94. Its exemption treatment was correct, but the omitted deductions raised taxable income from $65,593.83 to about $67,496." -us,scenario_114,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model wrongly allowed a fully phased-out $12,000 age deduction and applied a 5.3% top rate instead of Virginia's 5.75% rate. It also used $19,518 rather than $19,419.94 of itemized deductions and omitted the $1,730 exemptions, so both its taxable base and rate calculation were wrong." -us,scenario_114,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so it failed the required output contract." -us,scenario_114,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the $8,500 standard deduction instead of $19,419.94 of itemized deductions and misstated the aged exemption as $930 rather than $800. The correct total exemptions are $1,730, producing taxable income of $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model labeled a filer with $65,593.83 of Virginia taxable income a low-income retiree and treated deductions and the age exemption as eliminating liability. The deductions and $1,730 exemptions reduce income but leave a substantial taxable base subject to Virginia's brackets." -us,scenario_114,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model invented a $12,000 pension deduction, subtracted all $23,328 of Social Security after already constructing income without its taxable portion, used an obsolete $1,000 standard deduction, and omitted the $19,419.94 itemized deduction and $1,730 exemptions. It also submitted $2,453.71 despite its own arithmetic yielding $2,542.44." +us,scenario_111,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated about $16,600 of Social Security as taxable even though its own stated combined income of $30,940 is below the $32,000 joint-filer threshold, not above the $44,000 upper threshold. It then calculated zero taxable income after the standard deduction but replaced the resulting $0 tax with an unsupported $1,362 estimate." +us,scenario_111,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $5,405.50 of Social Security in AGI, but still correctly found that the standard deduction reduced taxable income and tax liability to $0. It then submitted $1,400 despite deriving no taxable income and identifying no tax or credit computation that produces that amount." +us,scenario_111,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated aged, blind, or disabled status plus low bank assets as sufficient for Washington Medicaid and asserted without calculation that disregards reduced $43,640 below the senior income limit. It also invoked the $65 earned-income disregard and work incentives even though the head has no earnings; the head instead qualifies through no Medicaid category." +us,scenario_111,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model conflated the MAGI expansion pathway with aged/blind/disabled pathways and assumed that Social Security exclusion and low assets created eligibility. The traced MAGI is 2.02 times FPL, above the relevant MAGI limits, and aged/blind/disabled status produces no separate qualifying category here." +us,scenario_111,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model assumed that blind and disabled status automatically opened an SSI-related Medicaid pathway despite zero SSI receipt, then treated listed medical expenses as a spend-down that established eligibility. PolicyEngine assigns no aged/blind/disabled or other Medicaid category to the head, so the resource-limit and vehicle-exclusion discussion never establishes eligibility." +us,scenario_111,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that the household has no wages, then contradicted that reasoning by imposing $1,046.85 of mandatory payroll tax on retirement income. Social Security retirement benefits and taxable IRA distributions are not employee payroll-tax bases, so the absence of employment compensation yields payroll_tax of $0." +us,scenario_111,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated income below Washington's 200% gross-income screen as categorical eligibility and then awarded the $24 monthly minimum. The household fails the applicable income eligibility determination, so it is ineligible rather than an eligible household whose calculated allotment is raised to the minimum." +us,scenario_111,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model treated blindness or disability as conferring Washington Medicaid eligibility regardless of income. Those statuses only open disability-related pathways; the spouse still fails their applicable eligibility tests and has medicaid_category NONE. +us,scenario_111,spouse_medicaid_eligible,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model incorrectly isolated the spouse's zero personal income and disregarded the income attributed under the applicable Medicaid household and spousal-income rules. The resulting household income is $43,640, and neither the 138% FPL expansion pathway nor a disability-based pathway applies." +us,scenario_111,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model asserted that blindness and disability satisfy the disability-related pathway without applying its financial eligibility test. At $43,640 of household income and 2.02 times FPL for MAGI purposes, the spouse qualifies through neither ABD nor adult-expansion Medicaid." +us,scenario_111,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model converted potential access to SSI-related or medically needy pathways into automatic eligibility based on blindness and disability. The spouse receives no SSI and fails every available Medicaid pathway, so disability does not override the failed financial tests." +us,scenario_111,spouse_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"The model concluded that resources were below the ABD limit by excluding the vehicle, then incorrectly declared the household's $43,640 income low enough for ABD Medicaid. Passing an asserted resource test does not cure the failed disability-pathway income test, and the spouse qualifies through no category." +us,scenario_111,spouse_medicaid_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model assumed unspecified exclusions reduce Social Security and IRA income enough to pass the ABD income test and then treated the spouse's share of income as independently qualifying. The applicable calculations leave the spouse outside both the disability-related pathways and MAGI expansion, with MAGI income at 2.02 times FPL." +us,scenario_111,spouse_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model reduced household MAGI to the $18,240 IRA distribution by omitting the income included in the engine's Medicaid calculation, producing a false result below 138% FPL. The engine calculation places the spouse at 2.02 times FPL, and zero personal income plus resources below an asserted limit does not independently establish ABD eligibility." +us,scenario_111,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated disability as automatically conferring Medicare eligibility regardless of age. The spouse is 49, and the prompt supplies no Social Security disability entitlement or other qualifying under-65 Medicare pathway, so the spouse is not Medicare eligible." +us,scenario_111,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model collapsed the under-65 Medicare pathway into a bare disability test. Disability alone is insufficient, and the spouse has no listed Social Security disability entitlement or other qualifying Medicare basis." +us,scenario_111,spouse_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model acknowledged that disability can lead to Medicare through Social Security disability benefits but then inferred that entitlement from the disability flag. Because unlisted benefit receipt and status are false and no Social Security disability entitlement is provided, the 49-year-old spouse does not qualify." +us,scenario_111,spouse_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model incorrectly described PolicyEngine's disability-based eligibility as automatic for every disabled person under 65. The modeled pathway requires a qualifying Medicare basis beyond the disability flag, which the spouse lacks." +us,scenario_111,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,The model treated inclusion of a disability-based Medicare pathway as equivalent to eligibility whenever disability is true. The spouse's age is 49 and no qualifying Social Security disability entitlement or other under-65 Medicare basis is listed. +us,scenario_111,spouse_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model explicitly asserted that PolicyEngine makes every disabled individual Medicare eligible regardless of age. PolicyEngine does not use the disability flag alone for this pathway, and the spouse has no listed qualifying Medicare entitlement." +us,scenario_112,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $13,000 of financial assistance as ordinary taxable income. Excluding that assistance yields $4,842.82 of AGI after the $3,000 capital-loss deduction, and the standard deduction reduces taxable income to zero rather than $1,743." +us,scenario_112,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived an EITC of about $453 from $5,923 × 7.65% and correctly found no phaseout, but then submitted $191 without any supporting computation. Its final value contradicts its own complete EITC derivation." +us,scenario_112,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated negative taxable income and zero federal income-tax liability as eliminating the EITC. The EITC is refundable and phases in from $5,923 of earned wages, producing $453.10 despite deductions and capital losses." +us,scenario_112,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly computed $5,923 × 7.65% as approximately $453 and correctly found no phaseout or investment-income disqualification, then replaced that result with an unsupported $50 estimate. Low earnings in the phase-in range generate the calculated $453.10 rather than justify a discretionary reduction." +us,scenario_112,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly used very low or negative AGI from capital losses to erase the childless EITC. The credit phases in on the $5,923 of wages, while the capital loss neither removes those wages from earned income nor triggers an investment-income disqualification." +us,scenario_112,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model stated the correct phase-in calculation of roughly $453 but then substituted $492 based on an inapplicable proximity-to-plateau estimate. At $5,923 of wages, the filer remains in the 7.65% phase-in segment, yielding $453.10." +us,scenario_112,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated farm rent as earned income and then treated the capital loss and negative AGI as canceling the EITC. The relevant earned-income base is $5,923 of wages, and applying the childless phase-in rate yields $453.10 without an investment-income disqualification." +us,scenario_112,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the $8,389 employer-sponsored insurance premium input from gross wages when determining EITC earned income. The supplied $5,923 gross wage amount remains the EITC earned-income base and produces $453.10." +us,scenario_112,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a 7.67% childless EITC phase-in rate instead of 7.65%. Applying the correct rate and statutory rounding to $5,923 of wages yields $453.10, not $454.29." +us,scenario_112,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly declared the filer ineligible without applying the childless-worker EITC pathway. A single 31-year-old with $5,923 of wages satisfies the age and earned-income conditions, and the phase-in calculation yields $453.10." +us,scenario_112,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the employer-sponsored insurance premium as fully offsetting gross wages for EITC purposes. The $5,923 wage input supplies positive earned income, so the childless EITC phases in to $453.10." +us,scenario_112,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the employer-sponsored insurance premiums from the stated gross wages and set EITC earned income to zero. Using the $5,923 wage base produces a childless EITC of $453.10." +us,scenario_112,federal_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly limited the phase-in calculation to the lower $4,843 AGI rather than the $5,923 of earned wages. In this phase-in-range case, the capital-loss-reduced AGI does not replace the positive earned-income base, so the credit is $453.10." +us,scenario_112,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model equated having no qualifying children with having no refundable credit and omitted the childless EITC pathway. At age 31 with $5,923 of wages, the filer receives $453.10 of EITC." +us,scenario_112,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a child or an explicitly listed credit input and failed to compute the childless EITC from wages. The age-31 filer’s $5,923 of earned income generates $453.10." +us,scenario_112,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly asserted that $5,923 was insufficient earned income for the EITC. Childless EITC eligibility begins with positive earned income, and this amount lies in the phase-in range, producing $453.10." +us,scenario_112,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model incorrectly treated low income as making the childless EITC minimal or zero and mixed the employer-premium figure into an ACA-style MAGI calculation. The EITC instead phases in at 7.65% on $5,923 of wages, yielding $453.10." +us,scenario_112,federal_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly identified the $5,923 wage base and calculated about $453, but then imposed a nonexistent reduction for the long-term capital loss. A capital loss does not reduce an otherwise calculated EITC through the investment-income limit, so the credit remains $453.10." +us,scenario_112,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model omitted the childless EITC without performing an eligibility or phase-in calculation. The age-31 single filer has $5,923 of earned wages, which generates $453.10 of refundable EITC." +us,scenario_112,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated low MAGI as a standalone Texas Medicaid pathway for a childless adult. A 31-year-old nondependent must first belong to an active eligibility category, and this head belongs to none." +us,scenario_112,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model inferred Medicaid eligibility directly from low earned income and minimal assets without identifying a Texas eligibility category. Assets do not establish a MAGI pathway, and this childless, nondependent adult qualifies through none of the available categories." +us,scenario_112,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly assumed that low income qualifies a single Texas adult for Medicaid. Texas has no general MAGI pathway for this 31-year-old childless, nondependent adult, whose Medicaid category is NONE." +us,scenario_112,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model falsely stated that Texas expanded Medicaid and applied expansion coverage to an adult below the poverty line. Texas has no expansion pathway for this childless 31-year-old, and the head satisfies no other Medicaid eligibility category." +us,scenario_112,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The answer asserts eligibility without applying Texas's categorical requirement. The head is a 31-year-old nondependent who qualifies through no Medicaid category, so the low MAGI level and eligible immigration status are insufficient." +us,scenario_112,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly identified that no pregnancy, postpartum, breastfeeding, infant, or under-five status was provided, but then treated low income as sufficient for eligibility. WIC's demographic categorical requirement is mandatory, and the prompt makes every unlisted status false, so the head is not eligible regardless of passing the income test." +us,scenario_112,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model treated the entire $8,389 employer-sponsored insurance premium as the employee's pretax cafeteria-plan deduction, even though the prompt did not identify any employee salary-reduction contribution. The full $5,923 of wages remains subject to Social Security and Medicare taxes, yielding $453.10." +us,scenario_112,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the listed employer-sponsored insurance premiums from FICA wages. Those premiums are not stated employee pretax contributions, so Social Security and Medicare taxes apply to all $5,923 of wages and total $453.10." +us,scenario_112,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model converted the employer-sponsored insurance premium into an unstated pretax payroll deduction and reduced FICA wages to zero. The wage base is $5,923, producing $367.22 of Social Security tax and $85.88 of Medicare tax." +us,scenario_112,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model selected the correct $5,923 wage base and the correct 7.65% combined rate but computed or rounded the product incorrectly. The separately rounded components are $367.22 and $85.88, which total $453.10, not $455." +us,scenario_112,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's stated components do not add to its submitted answer: $367.21 plus $85.88 equals $453.09, not $479.38. Using the engine's component rounding gives $367.22 plus $85.88, or $453.10." +us,scenario_112,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The zero answer omits employee FICA taxes on the head's $5,923 of wages. Those wages generate $367.22 of Social Security tax and $85.88 of Medicare tax, totaling $453.10." +us,scenario_112,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model treated the $1,920 farm rent entry as positive net self-employment income and applied the 92.35% adjustment and 15.3% rate directly to it. It failed to determine the household’s net self-employment earnings, which are nonpositive and therefore produce no self-employment tax." +us,scenario_112,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model correctly stated that passive real-estate farm rent is not subject to self-employment tax, then contradicted that rule by calculating tax as though the entire $1,920 were farm-operator self-employment earnings. The actual net self-employment tax base is zero because the household has no positive net self-employment earnings." +us,scenario_112,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model improperly combined farm rent, financial assistance, and the capital loss into a self-employment tax base. Financial assistance and capital gains or losses are not self-employment earnings, and the household’s relevant net earnings are nonpositive, so no self-employment tax applies." +us,scenario_112,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated the full $1,920 of farm rent as net self-employment income and applied the standard 92.35% and 15.3% factors without establishing positive net earnings from self-employment. The resulting tax base is zero, and its submitted $269 also does not match its own stated calculation of about $271." +us,scenario_112,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the ordinary gross-income screen as disqualifying despite categorical eligibility through TANF non-cash assistance. It also understated the applicable SNAP deductions, calculating net income near $1,429 instead of $1,269.19, which passes the net-income test." +us,scenario_112,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model ended the eligibility analysis at the ordinary gross-income limit and omitted categorical eligibility through TANF non-cash assistance. That pathway allows the household to proceed to the net-income and allotment calculations. +us,scenario_112,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,The model applied the ordinary 130% FPL gross-income test as an absolute bar and never applied categorical eligibility through TANF non-cash assistance. Its separate concern about the vehicle and assets is also inapposite under that eligibility pathway. +us,scenario_112,snap,claude-opus-4.8,llm_error,other,False,"The model excluded the $13,000 financial assistance from countable SNAP income and therefore calculated an allotment from artificially low net income. Including the assistance yields a $380.70 expected contribution, so the eligible household receives only the statutory minimum allotment of $287.68 annually." +us,scenario_112,snap,claude-opus-5,llm_error,other,False,"The model omitted the $13,000 financial assistance from countable SNAP income and consequently estimated a near-maximum benefit. With that income included, the expected contribution exceeds the maximum allotment and the eligible household receives the minimum allotment instead." +us,scenario_112,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model treated the ordinary 130% FPL gross-income threshold as controlling even though TANF non-cash assistance establishes categorical eligibility. The household then passes the applicable net-income test and qualifies for the minimum allotment. +us,scenario_112,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model used the ordinary gross-income limit to deny SNAP and omitted the TANF non-cash categorical-eligibility pathway. It also stated that no deductions offset the income, whereas the SNAP deductions reduce net income to $1,269.19, below the applicable limit." +us,scenario_112,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model incorrectly made the 130% FPL gross-income test dispositive. TANF non-cash assistance confers categorical eligibility, after which the household passes the net-income test and receives the minimum allotment." +us,scenario_112,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model calculated SNAP net income as $1,440.20 by failing to apply the full set of engine-recognized deductions. The correct net income is $1,269.19, or 97% of the $1,304.17 guideline, so the net-income test is satisfied." +us,scenario_112,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The zero answer implies the model omitted categorical eligibility through TANF non-cash assistance and the resulting minimum-allotment rule. The household passes the net-income test and receives $287.68 annually. +us,scenario_112,snap,gemini-3.1-pro-preview,llm_error,other,False,"The model incorrectly treated an expected contribution above the maximum allotment as producing no benefit. Once eligible, this one-person household receives the minimum allotment even when the ordinary maximum-minus-contribution formula is zero." +us,scenario_112,snap,gemini-3.5-flash,llm_error,other,False,"The model both overstated net income at roughly $1,440 and incorrectly floored the allotment at zero when 30% of net income exceeded the maximum. Net income is $1,269.19, and the eligible household receives the minimum allotment rather than zero." +us,scenario_112,snap,gemini-3.6-flash,llm_error,other,False,The model incorrectly concluded that an expected contribution exceeding the maximum allotment forces the benefit to zero. SNAP instead pays the minimum allotment to this eligible one-person household. +us,scenario_112,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model applied only the standard and earned-income deductions and therefore overstated net income at about $1,434. The complete SNAP calculation yields $1,269.19, below the $1,304.17 net-income limit." +us,scenario_112,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model's unsupported resource-and-income shortcut omitted categorical eligibility through TANF non-cash assistance. The household passes the net-income test and is entitled to the minimum allotment. +us,scenario_112,snap,gpt-5.4-nano,llm_error,other,False,"The model ignored the prompt's explicit one-person household composition and supplied zero instead of performing the SNAP tests. The stated facts establish categorical eligibility, net income below the limit, and an annual minimum allotment of $287.68." +us,scenario_112,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated gross countable income above the ordinary limit as disqualifying and omitted categorical eligibility through TANF non-cash assistance. The household proceeds past that screen, passes the net-income test, and receives the minimum allotment." +us,scenario_112,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly used the ordinary one-person gross-income limit as an absolute eligibility bar. TANF non-cash assistance establishes categorical eligibility, and the household qualifies for the minimum allotment." +us,scenario_112,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model understated SNAP deductions by reducing income only for the standard and earned-income deductions and assuming no further applicable deduction. The full calculation produces net income of $1,269.19, which passes the $1,304.17 limit." +us,scenario_112,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model incorrectly treated countable income above the ordinary gross-income threshold as dispositive. Categorical eligibility through TANF non-cash assistance prevents that gross test from disqualifying the household. +us,scenario_112,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model's generic income-and-assets conclusion omitted categorical eligibility through TANF non-cash assistance. Assets are only $10, net income passes the applicable limit, and eligibility triggers the minimum allotment." +us,scenario_112,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model collapsed two distinct tests into a generic denial: categorical eligibility prevents the ordinary gross-income screen from barring the household, and the correctly calculated $1,269.19 net income passes the net-income limit. The household therefore receives the minimum allotment." +us,scenario_112,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model denied eligibility solely because income exceeded its estimated ordinary gross-income limit. It failed to apply categorical eligibility through TANF non-cash assistance, under which the household proceeds to and passes the net-income test." +us,scenario_112,snap,inkling,llm_error,categorical_eligibility,False,"The model incorrectly treated the 130% FPL gross-income threshold as an absolute bar. TANF non-cash assistance confers categorical eligibility, and the household then qualifies for the minimum allotment." +us,scenario_112,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model applied the ordinary gross-income requirement for a nonelderly, nondisabled household while omitting categorical eligibility through TANF non-cash assistance. That pathway removes the asserted gross-income disqualification." +us,scenario_112,snap,kimi-k3,llm_error,categorical_eligibility,False,The model explicitly assumed the household lacked TANF categorical eligibility even though the financial-assistance input produces TANF non-cash categorical eligibility in the engine. It also treated an expected contribution above the maximum as eliminating the allotment instead of triggering the minimum benefit. +us,scenario_112,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly inferred that a single wage earner cannot qualify and did not perform the stated SNAP eligibility calculation. This one-person household is categorically eligible through TANF non-cash assistance, passes the net-income test, and receives the minimum allotment." +us,scenario_112,snap,qwen-3.7-max,llm_error,other,False,"The model's final $3,576 estimate contradicts its own calculation that countable financial assistance makes the ordinary benefit formula nonpositive. The correct next step is to apply the minimum-allotment rule to the eligible household, producing $287.68 rather than a near-maximum benefit." +us,scenario_112,snap,qwen3.8-max,llm_error,categorical_eligibility,False,The unexplained zero omits categorical eligibility through TANF non-cash assistance and the minimum-allotment rule. The household passes the net-income test and receives $287.68 annually. +us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions by using only listed property tax for SALT and its own medical calculation, producing $19,518 instead of the engine-calculated $24,787.23. It then abandoned its own $11,889 bracket calculation and submitted $8,971 without any valid tax computation." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted taxable Social Security from gross income, treated the QBI deduction as an AGI adjustment without actually subtracting it, and used a fabricated $28,300 standard deduction. It also improperly added employee Medicare tax and an unexplained withholding adjustment to federal income tax." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model capped the senior deduction at an unphased $6,000 instead of applying the $4,105.65 income-phased amount and misstated the medical and SALT components of itemization. Its submitted $5,489 also contradicts its own recomputed tax of about $10,711." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented $4,500 of mortgage interest from the mortgage balance and omitted the additional senior deduction entirely. Those errors produced taxable income of $77,683 instead of $72,807.33, followed by an unsupported reduction from its own $11,802 calculation to $11,210." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted half of self-employment tax from AGI for the partnership/S-corp income and selected a standard-deduction calculation instead of the engine's larger itemized deduction pathway. It then reduced an estimated $11,900 regular tax to $6,900 without identifying any nonrefundable credit." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented $6,693 of mortgage interest from the outstanding balance and omitted the $4,105.65 additional senior deduction. Its resulting taxable income of $75,490 therefore did not follow the engine's $33,765.23 total deduction calculation." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model used only $19,518 of itemized deductions and omitted the income-phased $4,105.65 additional senior deduction. This left taxable income at $82,183 rather than $72,807.33 and overstated tax." +us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model calculated only $21,069.57 of itemized deductions and omitted the $4,105.65 additional senior deduction. That raised taxable income to $80,630.83 instead of $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied obsolete 10%, 15%, and 25% brackets and a personal exemption rather than the governing 2026 rate schedule and additional senior deduction. It also invented a half-self-employment-tax AGI adjustment for the partnership/S-corp income." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies taxable income near $81,300 because it failed to apply the full $33,765.23 deduction stack. The correct deduction stack reduces AGI of $106,572.56 to taxable income of $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model explicitly applied pre-TCJA personal-exemption law instead of the governing 2026 provisions. It consequently omitted the current QBI and additional senior deduction treatment and used the wrong tax schedule. +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used sunset-era brackets and a $5,050 personal exemption instead of the governing 2026 rate schedule and $4,105.65 additional senior deduction. It also understated itemized deductions, leaving taxable income above the correct $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used $22,018.04 of itemized deductions and omitted the $4,105.65 additional senior deduction. This produced taxable income of $79,682.36 rather than $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model incorrectly assumed the QBI deduction disappeared and personal exemptions plus 10%, 15%, and 25% brackets returned in 2026. The applicable computation retains the $4,872.35 QBI deduction, uses the $4,105.65 senior deduction, and applies the governing 2026 brackets." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the deductions as sufficient to eliminate more than $106,000 of AGI. After the allowable $33,765.23 deduction stack, taxable income remains $72,807.33, so regular federal income tax is not zero." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model excluded or sheltered taxable retirement and Social Security income merely because the taxpayer is age 69. The income rules produce $106,572.56 of AGI and $72,807.33 of taxable income after deductions, not zero taxable income." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a post-2025 personal exemption and regular brackets that do not govern this 2026 calculation. It also understated deductions at about $22,873 including the exemption, instead of applying the $33,765.23 stack containing itemization, QBI, and the senior deduction." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model improperly deducted half of estimated self-employment tax from AGI and combined standard, senior, and QBI deductions to reach taxable income near $67,020. The correct pathway itemizes and produces taxable income of $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model chose the standard-deduction pathway and added a non-itemizer charitable deduction, missing that the engine's itemized deduction pathway is more favorable. With itemization, QBI, and the phased senior deduction, taxable income is $72,807.33 rather than about $78,445." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The submitted tax implies that the model did not reproduce the full $33,765.23 deduction stack or the resulting $72,807.33 taxable income. In particular, the age-based amount is the income-phased $4,105.65 additional senior deduction, not a generic age-65 addition applied without the engine's itemization calculation." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The unexplained $10,234 does not follow from the traced taxable income of $72,807.33. Applying the 2026 brackets to that taxable income after the full itemized, QBI, and senior deductions yields $10,729.61." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied restored 10%, 15%, and 25% brackets and a personal exemption instead of the governing 2026 schedule and additional senior deduction. It also understated the total deduction stack, producing taxable income of $79,963 instead of $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used $22,264 of itemized deductions and omitted the $4,105.65 additional senior deduction. That left taxable income at $79,437 instead of $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model wrongly reduced AGI for half of self-employment tax, claimed a personal exemption, and applied pre-TCJA brackets. The governing calculation instead uses AGI of $106,572.56, the QBI and phased senior deductions, and current 2026 rates." +us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model selected the standard deduction after calculating itemized deductions as only $17,518 and then added a non-itemizer charitable deduction. The engine's itemized deduction is $24,787.23, so itemization plus QBI and the $4,105.65 senior deduction reduces taxable income to $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model acknowledged positive taxable income but then reduced the liability to zero without identifying any available nonrefundable credit. The deductions leave $72,807.33 taxable, and no credit erases the resulting $10,729.61 tax." +us,scenario_114,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted taxable Social Security of $19,828.80, ignored the larger itemized-deduction pathway and the $4,105.65 senior deduction, and misstated the age-based standard deduction. Its asserted $15,654 tax also does not follow from its own stated taxable income of $66,122 under the 2026 brackets." +us,scenario_114,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model arithmetically understated AGI by more than $26,000 despite listing the income components that produce about $106,573. It then used an invented $20,400 standard deduction, misstated the QBI deduction, omitted the itemized and senior-deduction pathway, and submitted $5,290.25 despite stating regular tax of $7,761.45." +us,scenario_114,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model applied the childless EITC phaseout using only $22,700 of wages and ignored the rule requiring the greater of earned income or adjusted gross income. The filer’s other taxable income raises AGI beyond the childless EITC phaseout ceiling, yielding EITC of $0 rather than $295." +us,scenario_114,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, so it failed the required submission contract." +us,scenario_114,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model correctly calculated both employee payroll-tax components and correctly excluded self-employment tax, but submitted $3,470.30 instead of their $1,736.55 sum. Its answer effectively duplicates the employee payroll-tax total despite its own reasoning identifying no additional component." +us,scenario_114,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly derived $1,736.55, then replaced it with $1,481.30 through an unsupported “program simplification/rounding” adjustment. No rounding or simplification changes the exact $1,407.40 and $329.15 components." +us,scenario_114,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." +us,scenario_114,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model correctly summed federal employee Social Security and Medicare taxes to $1,736.55, then added $39.15 as a purported Virginia state payroll tax. Virginia contributes no mandatory employee payroll-tax component to this output." +us,scenario_114,self_employment_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model treated the $24,362 partnership or S-corporation income input as self-employment earnings. That amount does not enter the self-employment tax base here; unlisted self-employment income is zero, yielding $0." +us,scenario_114,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly placed the $24,362 partnership or S-corporation income in the self-employment tax base, then compounded the error by applying the 92.35% adjustment twice. The applicable self-employment earnings input is zero, so no self-employment tax arises." +us,scenario_114,self_employment_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model classified all $24,362 of partnership or S-corporation income as self-employment earnings and assessed both Social Security and Medicare components on it. That income is outside the self-employment tax base in this household, whose self-employment income is zero." +us,scenario_114,self_employment_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model assumed the combined partnership or S-corporation income represented a general partner’s self-employment income, even though no general-partner or self-employment status was listed. Unlisted self-employment income and status are zero or false, so the self-employment tax base is zero." +us,scenario_114,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model explicitly inferred that the $24,362 was active partnership income subject to self-employment tax, contrary to the instruction not to infer unlisted status. The combined partnership or S-corporation income input does not establish net earnings from self-employment, leaving a zero tax base." +us,scenario_114,self_employment_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model equated the $24,362 partnership or S-corporation income input with self-employment income and applied 92.35% and 15.3% to it. The household has zero net earnings from self-employment, so those factors apply to zero." +us,scenario_114,self_employment_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model applied self-employment tax directly to the $24,362 partnership or S-corporation income. That input is not part of this household’s self-employment tax base, which is zero." +us,scenario_114,self_employment_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model first misclassified the $24,362 partnership or S-corporation income as self-employment earnings, then introduced nonexistent adjustments for an 80% deduction limitation and a Virginia tax offset. Neither adjustment determines self-employment tax liability, and the actual self-employment tax base is zero." +us,scenario_114,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model treated the $24,362 partnership or S-corporation income as net self-employment income and taxed 92.35% of it. No self-employment income is listed, so the relevant base is zero rather than $24,362." +us,scenario_114,self_employment_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model applied self-employment tax to the entire $24,362 partnership or S-corporation income amount. That amount does not constitute net earnings from self-employment in this case, so the liability is $0." +us,scenario_114,self_employment_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model used the standard 92.35%-of-income calculation on the wrong income category by treating partnership or S-corporation income as self-employment income. With no listed self-employment earnings, the taxable base is zero." +us,scenario_114,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"The answer implies that the model taxed the $24,362 partnership or S-corporation income after unspecified deductions. This household has no self-employment earnings to which such deductions or the self-employment tax rate could apply, yielding $0." +us,scenario_114,self_employment_tax,inkling,llm_error,payroll_tax_base,False,"The model converted the $24,362 partnership or S-corporation income into net earnings using the 92.35% factor. That income is not self-employment earnings for this calculation, so the conversion starts from zero and produces no tax." +us,scenario_114,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax value or explanation, violating the required output contract." +us,scenario_114,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly treated the $24,362 partnership or S-corporation income as self-employment income and computed tax on 92.35% of it. Its submitted $3,459.40 also contradicts its own stated calculation of $3,442.62; the correct base for this output is zero." +us,scenario_114,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model incorrectly classified the $24,362 partnership or S-corporation income as self-employment earnings. It also added a purported amount above the Social Security wage base even though wages plus its assumed earnings were far below that base; with zero self-employment earnings, neither component applies." +us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived tax near $3,555 from its own taxable-income estimate and then submitted $2,597 without identifying any applicable credit or adjustment. It failed to carry its calculation through to the submitted value and also used $19,518 rather than the traced $19,419.94 itemized deduction." +us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted charitable contributions and employee expenses in computing federal AGI, omitted pension, IRA, partnership, and taxable Social Security income from its final AGI arithmetic, and invented a $28,000 Virginia standard deduction. Those errors erased taxable income that instead equals $65,593.83 after Virginia adjustments." +us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's stated computation produces about $4,227, but it submitted $1,854 without naming any deduction or nonrefundable credit that bridges the difference. It also used the standard deduction instead of the larger $19,419.94 itemized deduction and omitted the $1,730 exemptions from taxable income." +us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model recognized that itemizing was preferable but never calculated the listed deductions, substituting an unsupported estimate and returning $4,014. The traced itemized deduction is $19,419.94 and exemptions are $1,730, producing taxable income of $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used a standard deduction and then treated personal exemptions as credits. Virginia instead subtracts $19,419.94 of itemized deductions and $1,730 of exemptions from Virginia AGI before applying the brackets." +us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated the $930 personal exemption and $800 aged exemption as dollar-for-dollar tax credits. They are deductions from Virginia AGI, alongside the $19,419.94 itemized deduction, so subtracting $1,730 from tax understated liability sharply." +us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded all Social Security from federal AGI, granted a fully phased-out $12,000 age deduction, and failed to subtract the actual $1,730 exemptions. It also derived tax near $2,948 but submitted $2,450 with no supporting computation." +us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model granted a $12,000 age deduction that is eliminated by the income limitation and used the standard deduction rather than $19,419.94 of itemized deductions. It also omitted the $1,730 personal and aged exemptions." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated deductions at $23,948.60 and subtracted only an $800 exemption. The traced amounts are $19,419.94 of itemized deductions and $1,730 of exemptions, yielding $65,593.83 of taxable income." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $4,180 implies taxable income around $77,174, consistent with using a standard deduction and omitting the larger listed itemized deductions. The required taxable-income computation produces $65,593.83 before applying Virginia's rates." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model named Social Security and itemized deductions but did not use the traced $19,419.94 deduction or the $1,730 exemptions. Its $3,726 answer corresponds to taxable income above the required $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $15,511 and used an unsupported $1,600 exemption total. Virginia allows $19,419.94 of itemized deductions and $1,730 of exemptions here, reducing taxable income from its $69,633 estimate to $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used only $17,518.04 of itemized deductions instead of $19,419.94. With the correctly stated $1,730 exemptions, that $1,901.90 deduction shortfall explains its excess taxable income and tax." +us,scenario_114,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The submitted $14,036.62 is incompatible with the model's own Virginia brackets and described income adjustments, which would produce tax near $4,000 even under its standard-deduction shortcut. It failed to execute the stated rate calculation and also omitted the larger $19,419.94 itemized deduction and $1,730 exemptions." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and exemptions fully shelter the income without calculating them. After the Social Security subtraction, itemized deductions, and exemptions, $65,593.83 remains taxable rather than zero." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the retirement-heavy income as effectively sheltered, despite taxable pension, IRA, partnership, wage, interest, and Social Security income producing federal AGI of $106,572.56. Virginia taxable income remains $65,593.83 after all applicable subtractions and deductions." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $17,518 rather than $19,419.94. Its exemptions were correct, so the omitted $1,901.94 of deductions caused its taxable income and tax to be too high." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model applied an unavailable age deduction and thereby reduced taxable income to about $51,647. The income-limited age deduction is zero; only the $800 aged exemption applies within the $1,730 exemption total, leaving $65,593.83 taxable." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used an $8,750 standard deduction instead of the larger $19,419.94 itemized deduction and omitted the $800 aged exemption. Correct deductions and total exemptions reduce taxable income to $65,593.83 rather than about $77,064." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The $4,658 answer implies taxable income around $85,487, effectively failing to apply most of the traced $19,419.94 itemized deduction and $1,730 exemptions. Virginia taxes $65,593.83 after those reductions." +us,scenario_114,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed the standard deduction and exemptions eliminate liability, but $65,593.83 remains taxable after the larger itemized deduction and all exemptions. Virginia's brackets therefore produce substantial tax, not zero." +us,scenario_114,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $17,518 instead of $19,419.94. That $1,901.94 shortfall raised its taxable income from $65,593.83 to roughly $67,496 and overstated tax." +us,scenario_114,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly granted a fully phased-out $12,000 age deduction and applied a 5.3% top rate instead of Virginia's 5.75% rate. It also omitted the $1,730 personal and aged exemptions, so both its tax base and rate computation were wrong." +us,scenario_114,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted only the $930 personal exemption and omitted the additional $800 aged exemption. Its approximate itemized deduction also differed from the traced $19,419.94, leading to taxable income below the required $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required output contract." +us,scenario_114,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the $8,500 standard deduction instead of $19,419.94 of itemized deductions and misstated the aged exemption as $930 rather than $800. The correct exemption total is $1,730, producing taxable income of $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model labeled a filer with more than $106,000 of federal AGI a low-income retiree and asserted deductions eliminated tax without computing them. After all Virginia subtractions, itemized deductions, and exemptions, taxable income is $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model improperly granted a $12,000 pension or age deduction despite its income phaseout, subtracted Social Security again after already excluding it from its reconstructed AGI, and used a $1,000 standard deduction. It also derived $2,542.44 but submitted $2,453.71 without a supporting adjustment." +us,scenario_114,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI at $80,164, granted an unavailable $12,000 age deduction, used the standard deduction instead of itemizing, and invented $745.50 of nonrefundable credits. The applicable reductions instead produce $65,593.83 of taxable income, with the bracket tax itself furnishing the requested amount." us,scenario_114,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_115,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented an Alabama aged/disabled Medicaid pathway for this person and treated Social Security of $18,708 as below that pathway's income limit. PolicyEngine found medicaid_category = NONE and SSI received = 0, so no aged/disabled or SSI-related eligibility pathway applies regardless of the model's countable-income shortcut." -us,scenario_115,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model counted Social Security plus tax-exempt interest and then concluded the household met an Alabama aged/disabled Medicaid pathway. It missed the categorical gate: PolicyEngine assigned no Medicaid category, so low assets and a monthly income calculation do not establish eligibility." -us,scenario_115,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model applied an SSI-related aged/disabled Medicaid pathway and assumed SSA exclusions and disregards reduce countable income below the qualifying threshold. PolicyEngine records SSI received as 0 and medicaid_category = NONE, so the SSI-related pathway never opens and the vehicle/assets discussion is immaterial." -us,scenario_115,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a parseable head_medicaid_eligible value or explanation. This is a missing-output failure rather than a substantive Medicaid-rule calculation. -us,scenario_115,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated age, disability, Social Security income after a general disregard, and low bank assets as sufficient for Alabama SSI-related Medicaid. PolicyEngine found no qualifying Medicaid category and zero SSI receipt, so the person is not eligible despite satisfying immigration status and having MAGI at 1.47 FPL." +us,scenario_115,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented an Alabama aged/disabled Medicaid pathway for the head instead of determining whether PolicyEngine assigned an eligible category. The engine assigns medicaid_category = NONE, so comparing $18,708 of Social Security income with an asserted aged/disabled limit cannot produce eligibility." +us,scenario_115,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age, disability, low assets, and $1,953 of monthly income as sufficient for an Alabama aged/disabled Medicaid category. PolicyEngine assigns no Medicaid category, so its income-and-assets comparison was applied to a pathway the head does not qualify through." +us,scenario_115,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly placed the head in Alabama's SSI-related aged/blind/disabled pathway even though SSI received is zero and PolicyEngine assigns no Medicaid category. It also wrongly described the $18,708 Social Security benefit as largely excluded: the $20 general disregard does not reduce that income below the SSI-level threshold." +us,scenario_115,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for head_medicaid_eligible, violating the required output contract." +us,scenario_115,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model assumed that being aged and disabled automatically opened an SSI-related Alabama Medicaid pathway and then declared $1,559 of monthly Social Security income low enough after unspecified deductions. PolicyEngine assigns medicaid_category = NONE, so the head has no qualifying pathway and the asset comparison does not establish eligibility." us,scenario_115,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_115,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded $4,734 of tax-exempt interest from SNAP income, reducing annual income from approximately $23,442 to $18,708. Its own allotment calculation produced $0, but it then invented a positive minimum benefit and ultimately submitted $2,496 without a computation supporting that amount." -us,scenario_115,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted the $4,734 of tax-exempt interest and imported a $20 general-income exclusion from SSI methodology into SNAP. It also treated $250 of annual medical expenses as creating a substantial deduction even though the SNAP medical threshold is applied monthly, then asserted a near-maximum allotment without computing the income-based reduction." -us,scenario_115,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model excluded $4,734 of tax-exempt interest and therefore understated SNAP income. Its stated formula still produced a benefit below zero, but it replaced that result with an unsupported $142 monthly allotment; minimum-benefit rules do not convert an income-ineligible household into a recipient." -us,scenario_115,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 of tax-exempt interest, yet its own net-income and maximum-allotment calculation still yielded $0. It then overrode that result with an invented $321 monthly minimum benefit, even though minimum-benefit treatment does not create eligibility and $321 exceeds the one-person maximum allotment it cited." -us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly excluded the $4,734 interest amount from Alabama AGI. Alabama AGI is $4,734; after the $3,000 standard deduction and $1,500 personal exemption, $234 remains taxable and produces $4.68 of tax." -us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest amount and invoked unspecified elderly exemptions and nonrefundable credits to eliminate the liability. The applicable deductions leave $234 of Alabama taxable income, with no nonrefundable credit reducing the resulting $4.68 tax." -us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model correctly exempted Social Security but incorrectly treated the $4,734 interest amount as excluded from Alabama income. That interest forms Alabama AGI and leaves $234 taxable after the applicable deductions." -us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model incorrectly set Alabama AGI to zero by excluding the $4,734 interest amount. Alabama AGI is $4,734, and the applicable deductions reduce it only to $234 rather than zero." -us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model incorrectly excluded the $4,734 interest amount from Alabama taxable income. Alabama includes it in AGI, leaving $234 taxable after the $4,500 combined standard deduction and personal exemption." -us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model correctly exempted Social Security but incorrectly excluded the $4,734 interest amount from Alabama gross income. Including that amount and applying the $3,000 standard deduction and $1,500 personal exemption leaves $234 taxable." -us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly treated both income sources as outside Alabama taxable income. Only Social Security is excluded here; the $4,734 interest amount enters Alabama AGI and generates $4.68 of tax after deductions." -us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model incorrectly excluded the $4,734 interest amount and therefore set state taxable income to zero. Alabama's calculation includes that amount in AGI, leaving $234 taxable after deductions." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used an incorrect deduction-and-exemption total of at least $5,000. The applicable $3,000 standard deduction and $1,500 personal exemption total $4,500, leaving $234 of the $4,734 Alabama AGI taxable." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the taxable amount fell below the zero-tax threshold. The $4,734 Alabama AGI exceeds the applicable $4,500 combined standard deduction and personal exemption by $234, which is taxed at 2%." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model exempted Social Security and then incorrectly concluded that no taxable income remained, omitting the $4,734 interest amount from Alabama AGI. That amount leaves $234 taxable after deductions." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model incorrectly excluded the $4,734 interest amount from Alabama income. Alabama includes it in AGI, and the applicable deductions leave $234 subject to the 2% rate." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model focused only on Alabama's Social Security exemption and omitted the $4,734 interest amount that forms Alabama AGI. After deductions, that amount leaves $234 taxable and $4.68 due." -us,scenario_115,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model incorrectly treated the $4,734 interest amount as federally exempt interest that Alabama also excludes. Alabama includes the amount in AGI, so AGI is $4,734 rather than zero and taxable income is $234." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly concluded that the absence of wages meant there was no Alabama-taxable income. The $4,734 interest amount enters Alabama AGI and leaves $234 taxable after the applicable deductions." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced taxable income to zero after deductions. Alabama AGI is $4,734 and the applicable $4,500 combined standard deduction and personal exemption leave $234 taxable, producing $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model correctly exempted Social Security but incorrectly asserted that the available deductions eliminated all other Alabama income. The $4,734 Alabama AGI exceeds the $4,500 combined standard deduction and personal exemption by $234." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model incorrectly equated the absence of federal taxable income with the absence of Alabama taxable income. Alabama separately includes the $4,734 interest amount in AGI, leaving $234 taxable after state deductions." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model incorrectly concluded that the reported interest produced no Alabama taxable income. The $4,734 interest amount forms Alabama AGI and exceeds the applicable $4,500 deductions by $234." -us,scenario_115,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model correctly applied Alabama's Social Security exemption but incorrectly stated that there was no other taxable income. The $4,734 interest amount enters Alabama AGI and yields $234 of taxable income after deductions." -us,scenario_115,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model recognized that Alabama could add the interest amount but incorrectly concluded that deductions fully offset it. The applicable standard deduction and personal exemption total $4,500, not enough to eliminate $4,734 of AGI, so $234 remains taxable." -us,scenario_115,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that exemptions and deductions were sufficient to eliminate the $4,734 interest amount if included. The applicable $4,500 combined standard deduction and personal exemption leave $234 taxable." -us,scenario_115,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model incorrectly treated the $4,734 interest amount as exempt from Alabama income tax. Alabama includes it in AGI, and $234 remains taxable after the applicable deductions." -us,scenario_115,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly considered adding the $4,734 interest amount back but used an incorrect deduction-and-exemption total of at least $5,500. The applicable $3,000 standard deduction plus $1,500 personal exemption totals $4,500, leaving $234 taxable." -us,scenario_115,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model made the categorical factual error that Alabama has no state income tax. Alabama imposes an individual income tax, and the household's $234 of taxable income is taxed at 2%, producing $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model correctly excluded Social Security but incorrectly excluded the $4,734 interest amount from Alabama taxable income. Alabama includes that amount in AGI, leaving $234 taxable after the applicable deductions." +us,scenario_115,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model excluded the $4,734 of tax-exempt interest and waived the gross-income eligibility test based on age and disability. PolicyEngine applies the applicable income limit to approximately $23,442 of household income, producing ineligibility before any medical deduction or minimum-allotment calculation." +us,scenario_115,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model treated only Social Security as countable, invented a SNAP $20 general deduction, and proceeded directly to a net-benefit calculation without applying the eligibility threshold to total income of approximately $23,442. That total exceeds the one-person limit, so no allotment is calculated." +us,scenario_115,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model waived the gross-income test, excluded tax-exempt interest, and converted a negative allotment formula into an unsupported $142 monthly benefit. Total income of approximately $23,442 exceeds the applicable eligibility limit, and minimum-benefit rules cannot override that ineligibility." +us,scenario_115,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly computed that the ordinary allotment formula yields $0 even after excluding tax-exempt interest, but then replaced that result with an unsupported $321 monthly minimum benefit. It also missed that approximately $23,442 of income exceeds the applicable SNAP eligibility threshold; minimum-allotment rules do not grant benefits after failure of eligibility." +us,scenario_115,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced all countable income to zero through a standard medical deduction. The listed $250 of annual medical and over-the-counter expenses cannot erase approximately $23,442 of annual household income, which exceeds the applicable one-person SNAP eligibility limit." +us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest from Alabama AGI. Alabama's $3,000 standard deduction and $1,500 personal exemption leave $234 taxable, producing $4.68 at the 2% bracket." +us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model excluded the $4,734 interest and invoked unspecified elderly exemptions and nonrefundable credits that do not eliminate the liability. The applicable $4,500 of standard deduction and personal exemption leaves $234 taxable and $4.68 due." +us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model correctly exempted Social Security but incorrectly excluded the $4,734 interest from Alabama AGI. After $4,500 in applicable deductions and exemptions, $234 remains taxable." +us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly set Alabama AGI to zero by excluding the $4,734 interest. Alabama AGI is $4,734 here, and the $4,500 deduction-and-exemption total leaves $234 taxable." +us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $4,734 interest as excluded from Alabama taxable income. The interest enters Alabama AGI, leaving $234 taxable after the $3,000 standard deduction and $1,500 personal exemption." +us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model correctly excluded Social Security but incorrectly excluded the $4,734 interest from Alabama gross income. That interest exceeds the applicable $4,500 standard deduction and personal exemption by $234, generating $4.68 of tax." +us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated both income sources as excluded when the $4,734 interest enters Alabama AGI. The applicable deductions and exemption total $4,500, not enough to eliminate the resulting income." +us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest from Alabama AGI. Subtracting the $4,500 standard deduction and personal exemption leaves $234 taxable at 2%." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated the deductions and exemptions as at least $5,000. The applicable standard deduction is $3,000 and the personal exemption is $1,500, leaving $234 of the $4,734 Alabama AGI taxable." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that Alabama taxable income fell below the tax threshold. The $4,734 Alabama AGI exceeds the applicable $4,500 standard deduction and personal exemption by $234." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated the Social Security exemption as eliminating all Alabama income and omitted the $4,734 interest from Alabama AGI. That interest leaves $234 taxable after deductions and exemptions." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest from Alabama taxable income. Alabama AGI includes it, and $234 remains after the $4,500 deduction-and-exemption total." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model stopped after applying Alabama's Social Security exemption and ignored the $4,734 interest included in Alabama AGI. After $4,500 in deductions and exemptions, $234 remains taxable." +us,scenario_115,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly classified the $4,734 interest as excluded from Alabama AGI. Alabama AGI is $4,734 here, leaving $234 taxable after the standard deduction and personal exemption." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed that the absence of wages and the tax-exempt label left no Alabama-taxable income. The $4,734 interest enters Alabama AGI and produces $234 of taxable income after $4,500 in deductions and exemptions." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced taxable income to zero. The $4,734 Alabama AGI exceeds the $3,000 standard deduction plus $1,500 personal exemption by $234, yielding $4.68 of tax." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly stated that no Alabama-taxable income remained after deductions. The $4,734 interest is included in Alabama AGI, and the applicable $4,500 deduction-and-exemption total leaves $234 taxable." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model improperly inferred Alabama taxable income from the absence of federal taxable income. Alabama AGI includes the $4,734 interest, leaving $234 taxable after Alabama's deductions and exemption." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the reported interest produced no Alabama-taxable income. It enters Alabama AGI, and $234 remains after the $3,000 standard deduction and $1,500 personal exemption." +us,scenario_115,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model correctly exempted Social Security but incorrectly treated the household as having no other taxable income. The $4,734 interest enters Alabama AGI and leaves $234 taxable after $4,500 in deductions and exemptions." +us,scenario_115,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that Alabama's deductions fully offset the interest. The applicable $3,000 standard deduction and $1,500 personal exemption total $4,500, leaving $234 of the $4,734 Alabama AGI taxable." +us,scenario_115,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed that the deductions and exemptions were sufficient to eliminate the $4,734 interest. They total $4,500, leaving $234 taxable and producing $4.68 of Alabama tax." +us,scenario_115,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model correctly considered adding the interest to Alabama income but incorrectly concluded that deductions fully offset it. The $3,000 standard deduction and $1,500 personal exemption leave $234 taxable." +us,scenario_115,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest from Alabama taxable income along with Social Security. The interest enters Alabama AGI and exceeds the applicable deductions and exemption by $234." +us,scenario_115,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model overstated the combined standard deduction and personal exemption as at least $5,500. Their applicable total is $4,500, so $234 of the $4,734 Alabama AGI remains taxable." +us,scenario_115,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely stated that Alabama has no state income tax. Alabama imposes an individual income tax, and this household owes 2% of $234 in taxable income, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest from Alabama taxable income. With Social Security exempt but the interest included, $234 remains taxable after the $4,500 standard deduction and personal exemption." +us,scenario_115,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the standard deduction reduced Alabama taxable income to zero. The $3,000 standard deduction plus $1,500 personal exemption leaves $234 of the $4,734 Alabama AGI taxable." us,scenario_115,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_116,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated disability as a sufficient Florida Medicaid category and invented a disabled-adult income/resource pathway under PolicyEngine rules. PolicyEngine assigns the 42-year-old spouse `medicaid_category = NONE`, so the spouse never reaches an income-threshold test; the veterans benefits and bank assets do not rescue eligibility without a qualifying category." -us,scenario_116,spouse_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model assumed that being disabled at age 42 qualifies an adult for Medicaid under PolicyEngine rules. The correct derivation first checks categorical eligibility, and the spouse qualifies through none of Florida's pathways, so Medicaid eligibility is false before any income calculation matters." -us,scenario_116,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model converted the spouse's generic disability status into assumed SSDI entitlement and then assumed the 24-month Medicare waiting period was satisfied. The household facts list disability and veterans benefits, not SSDI entitlement or prior Medicare-qualifying disability duration, so the under-65 spouse does not meet the Medicare eligibility pathway." -us,scenario_116,spouse_medicare_eligible,qwen-3.7-max,llm_error,age_disability,False,The model treated veterans benefits as veterans disability benefits and then treated that inferred benefit as a Medicare-qualifying disability pathway. PolicyEngine's Medicare eligibility calculation does not make an under-65 spouse Medicare eligible from veterans benefits or a generic disability flag; it requires the age threshold or a specified Medicare-qualifying condition or entitlement pathway absent from the facts. -us,scenario_117,child3_early_head_start_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model recognized that household income exceeds typical Early Head Start limits but then ignored the income screen and submitted eligibility based only on Child 3 being age 2. It also inflated income by double-counting the FLSA overtime premium alongside wages, but the decisive error is that age under 3 is necessary and not sufficient for Early Head Start eligibility." -us,scenario_117,child3_early_head_start_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model applied only the Early Head Start age rule and omitted the low-income or categorical-eligibility requirement. Child 3 is in the under-age-3 service population, but the household's high income prevents eligibility." -us,scenario_117,child3_early_head_start_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model treated meeting the birth-through-age-2 age criterion as complete Early Head Start eligibility. It failed to apply the required family income or categorical eligibility test, which the household fails because its annual income is far above the poverty-based threshold." -us,scenario_117,child3_early_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model inferred eligibility solely from Child 3 being age 2. Early Head Start is limited to children under 3 who also satisfy the program's income or categorical pathway, and this household does not satisfy that non-age requirement." -us,scenario_117,child3_early_head_start_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model used Early Head Start's age range as a standalone eligibility rule. Child 3 remains under age 3, but the program's income-targeting rule still applies, and the household's income is too high for eligibility." -us,scenario_117,child3_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 2 as sufficient for Head Start eligibility and skipped the income-based eligibility screen. Under the PolicyEngine derivation, the household's roughly $250,733 income exceeds the Head Start poverty-guideline threshold, so Child 3 is not eligible." -us,scenario_117,child3_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age under 5 as sufficient for WIC eligibility and omitted the separate household income screen. Child 3 meets the child age category, but the household income of about $250,733 exceeds the 185% FPL WIC limit for a five-person household, so the correct eligibility result is no." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly phased down the $25,000 qualified-overtime deduction to $15,185 instead of applying the full cap, and it omitted the $2,000 charitable deduction available alongside the standard deduction. It also subtracted a $1,200 CDCC even though the head has no earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $35,525 overtime premium even though gross wages already include overtime, treated charity and employee expenses as AGI deductions, and omitted the qualified-overtime deduction. It then failed to subtract the $6,600 nonrefundable CTC from tax." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the mortgage balance and an assumed rate, then itemized that fabricated amount. It also included the state refund, used only $4,500 of child credits rather than $6,600, and allowed a $600 CDCC despite the head's zero earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The submitted $47,714 contradicts the model's own final derivation of approximately $25,972. The answer field therefore does not encode the calculation stated in its explanation." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model fabricated roughly $80,000 of itemized deductions from assumed mortgage interest and SALT, then stacked the overtime deduction on top. The correct deduction total is $59,200, and CDCC is zero because the head has no earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model included the $2,589 state refund in AGI, used a $30,000 standard deduction, and entirely omitted the $25,000 overtime deduction and $2,000 charitable deduction. It also subtracted $1,200 of CDCC despite the head having no earned income and used a $2,000 rather than $2,200 per-child CTC." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The submitted $62,700 contradicts the model's own calculation of $22,630. Its answer is therefore a contract-level mismatch between the numeric output and the stated derivation." -us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied obsolete 15%, 25%, and 28% brackets instead of the applicable 2026 rate schedule. It also omitted the $25,000 overtime deduction, $2,000 charitable deduction, and $6,600 CTC while incorrectly including the state refund and itemized deductions." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a pre-TCJA regime with personal exemptions and reduced the CTC to $3,000. The applicable computation instead uses the $32,200 standard deduction, $25,000 overtime deduction, $2,000 charitable deduction, current brackets, and a $6,600 CTC." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The $46,522 answer is incompatible with tax on $188,944.53 of taxable income followed by the $6,600 CTC. Its terse explanation implies it omitted the qualified-overtime and charitable deductions or failed to apply the full nonrefundable child credit." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model expressly used pre-TCJA 2026 rules rather than the applicable 2026 brackets and deductions. That choice omits the full $25,000 overtime deduction and $6,600 current-law CTC treatment that drive the reference result." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model incorrectly treated 2026 as a post-TCJA-expiration regime and declared a $1,000-per-child CTC fully phased out at this income. The household instead receives $2,200 for each of three children because AGI is below the $400,000 joint phaseout threshold." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model's $26,521.84 does not reflect the traced taxable income of $188,944.53 and the full $6,600 CTC. Its generic reference to a CTC phaseout is misplaced because this household is below the joint phaseout threshold." -us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included the state refund, used an approximate $31,000 standard deduction, and omitted both the $25,000 qualified-overtime deduction and $2,000 charitable deduction. Its rejection of CDCC was correct, but those taxable-income errors left liability overstated." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The $32,981 answer is consistent with taxing income after only a standard deduction and ordinary credits. The model failed to incorporate the full $25,000 overtime deduction and $2,000 charitable deduction that reduce taxable income to $188,944.53." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model incorrectly asserted that deductions and nonrefundable credits eliminate the liability. Tax before credits is $30,991.80 and the $6,600 CTC reduces it only to $24,391.80." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model substituted estimated Arkansas tax and medical itemization for the traced deduction stack and included the state refund in AGI. It omitted the $25,000 overtime deduction and $2,000 charitable deduction, so it taxed $214,996.05 rather than $188,944.53." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model applied only two child tax credits even though all three children are under 17 and generate $2,200 each. It also applied a dependent-care credit, which is zero because the head has no earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model incorrectly subtracted a $600 CDCC despite the head's zero earned income. With the traced deductions and $6,600 CTC as the only nonrefundable credit, liability is $24,391.80 rather than $24,361.48." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model itemized charitable and SALT deductions instead of using the traced $32,200 standard deduction plus the separate $25,000 overtime and $2,000 charitable deductions. That produced the wrong taxable-income base despite correctly recognizing the $6,600 CTC." -us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $52,000 estimate ignores the computation that reduces AGI to $188,944.53 of taxable income and then subtracts $6,600 of CTC. It effectively taxed the high gross wages without the full deduction and credit stack." -us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly reverted to personal exemptions, a 25% bracket, and a fully phased-out CTC. The applicable 2026 computation uses current brackets, no personal exemptions, the $25,000 overtime deduction, and a full $6,600 CTC." -us,scenario_117,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model improperly deducted the ESI premium from AGI and used a reverted regime with personal exemptions and 15% and 25% brackets. It also omitted the $25,000 overtime deduction and $6,600 CTC while subtracting a $1,200 CDCC that fails the earned-income requirement." +us,scenario_116,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model improperly awarded a SNAP minimum allotment after its own benefit calculation reduced the regular allotment below zero, even though the household fails the gross-income eligibility test. It also submitted $6,804 despite stating an annual result of $3,402, doubling its own incorrectly calculated amount." +us,scenario_116,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented expanded or special disabled-adult coverage and treated low assets and potentially favorable treatment of veterans benefits as sufficient. Florida assigns the spouse no Medicaid category, so neither the income comparison nor the $4,870 resource amount creates eligibility." +us,scenario_116,spouse_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the disability flag as an automatic Medicaid eligibility pathway. The spouse must first qualify under a Florida Medicaid category, and the engine assigns medicaid_category = NONE." +us,scenario_116,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model invented SSDI receipt and completion of Medicare's 24-month disability waiting period from the facts that the spouse is disabled and receives veterans benefits. The prompt makes unlisted benefit receipt false, so neither SSDI entitlement nor the waiting-period condition exists, leaving the 42-year-old spouse ineligible." +us,scenario_116,spouse_medicare_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model incorrectly treated veterans disability benefits as a direct Medicare eligibility pathway. Veterans benefits neither establish SSDI entitlement nor satisfy Medicare's under-65 disability requirements, so the spouse's age 42 and disability flag yield no Medicare eligibility." +us,scenario_117,child3_early_head_start_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model explicitly recognized that household income exceeded typical Early Head Start limits but then treated age 2 as sufficient for eligibility. It also incorrectly added the separately listed FLSA overtime premium to gross wages even though gross wages already include overtime, but the decisive error was failing to enforce the program's low-income test." +us,scenario_117,child3_early_head_start_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model applied only the under-age-3 requirement and omitted Early Head Start's low-income eligibility test. Child 3 satisfies the age condition, but household income of approximately $250,733 disqualifies the child." +us,scenario_117,child3_early_head_start_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model equated meeting the birth-through-age-2 criterion with full Early Head Start eligibility. It failed to apply the poverty-based household income condition, which the approximately $250,733 household income does not satisfy." +us,scenario_117,child3_early_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The answer implies that age 2 alone establishes Early Head Start eligibility. The correct eligibility sequence first confirms the child is under 3 and then applies the low-income test, which this approximately $250,733 household fails." +us,scenario_117,child3_early_head_start_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated remaining under age 3 throughout the year as sufficient to qualify as a participant. It omitted the required low-income condition, and this household's approximately $250,733 income is above the qualifying threshold." +us,scenario_117,child3_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 2 as sufficient for Head Start eligibility and omitted the income-eligibility test. The household's approximately $250,733 income exceeds the poverty-based Head Start limit, and no other qualifying pathway applies." +us,scenario_117,child3_wic_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated age under five as sufficient for WIC eligibility and omitted the separate income test. Although Child 3 meets the categorical age rule, household income exceeds the 185% federal-poverty-guideline threshold for a five-person household, yielding no WIC eligibility." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly phased down the $25,000 qualified-overtime deduction to $15,185; the trace uses the full $25,000 cap. It also subtracted a $1,200 CDCC that is not used, while omitting the $2,000 charitable deduction allowed alongside the standard deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $35,525 overtime premium even though gross wages already include overtime, then treated employee expenses and charity as AGI adjustments. It omitted the $25,000 overtime deduction, used the wrong joint standard deduction, and failed to subtract the actual $6,600 CTC." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the mortgage balance and an assumed interest rate, then itemized that invented amount. It also included the state refund in AGI, used outdated CTC amounts, and subtracted a CDCC absent from the trace instead of applying the traced standard, overtime, and charitable deductions." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its submitted $47,714 contradicts its own calculation near $25,972. Independently, that calculation invented mortgage interest and SALT deductions, included the state refund, omitted the $25,000 overtime deduction, and used $2,000 rather than $2,200 per child." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model invented roughly $80,000 of itemized deductions from an unlisted mortgage rate and SALT, then stacked the overtime deduction on top. The trace instead uses $59,200 total deductions and subtracts only the $6,600 CTC from pre-credit tax." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The final calculation included the state refund in AGI, used a $30,000 standard deduction, omitted the $25,000 overtime and $2,000 charitable deductions, and subtracted a $1,200 CDCC. The correct taxable income is $188,944.53 and the only traced nonrefundable credit is the $6,600 CTC." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The submitted $62,700 contradicts the model's own estimated post-credit tax of $22,630. Its underlying work also invented mortgage-interest and SALT deductions, included the state refund, omitted the qualified-overtime deduction, and used the wrong CTC amount." +us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied obsolete 15%, 25%, and 28% brackets instead of the applicable 2026 rate schedule. It also itemized invented SALT and medical deductions, omitted the overtime deduction, and incorrectly denied the unphased $6,600 CTC." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a pre-TCJA regime with personal exemptions and a reduced CTC instead of the applicable 2026 rules. It also invented SALT and medical deductions and omitted the $25,000 overtime deduction and $2,000 charitable deduction used with the standard deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies taxation of far more than the traced $188,944.53 of taxable income. The model failed to incorporate the combined $59,200 deduction package and the full $6,600 nonrefundable CTC needed to reach $24,391.80." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model explicitly used pre-TCJA 2026 rules rather than the applicable 2026 rate, deduction, and credit rules. That choice misses the $32,200 standard deduction, $25,000 overtime deduction, $2,000 charitable deduction, and $2,200-per-child CTC structure." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model wrongly applied post-TCJA-expiration rules and treated the CTC as fully phased out at roughly $248,000 of AGI. The household instead receives an unphased $6,600 nonrefundable CTC, and taxable income reflects the current-law overtime and charitable deductions." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer does not reproduce the traced taxable income of $188,944.53 or the resulting $30,991.80 pre-credit tax. Applying the full $6,600 CTC to that amount yields $24,391.80, not $26,521.84." +us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included the state refund, used an approximate $31,000 standard deduction, and omitted both the $25,000 overtime deduction and $2,000 charitable deduction. Those omissions inflated taxable income from $188,944.53 to about $219,734." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer implies that the model did not reduce AGI by the full traced $59,200 of deductions. Taxable income is $188,944.53, producing $30,991.80 before the $6,600 CTC and $24,391.80 afterward." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model incorrectly treated deductions and credits as sufficient to eliminate the liability. They reduce $248,144.53 of AGI to $188,944.53 of taxable income and then reduce $30,991.80 of tax by only $6,600, leaving $24,391.80." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model itemized estimated Arkansas tax and medical expenses, included the state refund, and omitted the $25,000 overtime deduction and $2,000 charitable deduction used with the standard deduction. Its denial of CDCC matches the trace, but its taxable-income derivation does not." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model counted only two child tax credits and also subtracted a dependent-care credit. All three children qualify for a $2,200 CTC, totaling $6,600, while no CDCC is used in the traced result." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model correctly identified the standard, overtime, charitable, and $6,600 CTC components but incorrectly subtracted a $600 CDCC. The trace subtracts no CDCC, and its remaining $30.32 discrepancy also reflects inaccurate bracket arithmetic." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model itemized charitable and SALT deductions instead of using the traced $32,200 standard deduction plus $25,000 overtime and $2,000 charitable deductions. It also failed to reproduce AGI of $248,144.53, which excludes the state refund and includes the $530.47 capital-loss deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $52,000 answer is consistent with taxing the high wages without the full traced deductions and credits. The required calculation reduces AGI by $59,200 and then subtracts the $6,600 CTC, leaving $24,391.80." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model wrongly used a reverted tax regime with personal exemptions and treated the CTC as fully phased out at $250,734 of AGI. It also invented state-tax and medical itemized deductions instead of applying the traced standard, overtime, and charitable deductions." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly deducted an employer-sponsored insurance premium from wages and applied reverted 15% and 25% brackets with personal exemptions. It also denied the $6,600 CTC and omitted the $25,000 qualified-overtime deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model correctly used the overtime cap and $6,600 CTC but substituted invented medical, SALT, and miscellaneous itemized deductions for the traced $32,200 standard deduction and $2,000 charitable deduction. It also started from the wrong AGI, so its taxable income and bracket tax were understated." us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model included the state refund and replaced the traced deduction stack with itemized Arkansas tax, medical expenses, and an overstated charitable amount. The correct base uses AGI of $248,144.53 and deductions of $32,200, $25,000, and $2,000, yielding taxable income of $188,944.53." -us,scenario_117,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model incorrectly claimed that credits offset all federal tax. The pre-credit tax is $30,991.80 and the available $6,600 nonrefundable CTC leaves $24,391.80 unpaid." -us,scenario_117,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the loan balance and an assumed 6.5% rate, added unsupported SALT, and itemized those amounts. It also omitted the $25,000 overtime deduction and $2,000 charitable deduction and subtracted a $600 CDCC despite the head's zero earned income." -us,scenario_117,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model invented a $3,750 refundable Child Tax Credit by applying the Additional CTC earned-income formula directly and then imposing an unsupported cap. It also inflated earned income by adding the FLSA overtime premium to wages even though gross wages already include overtime under the prompt, and it failed to carry the high-income CTC phaseout and tax-liability allocation through to a zero refundable CTC." -us,scenario_117,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not submit a parseable value or explanation for federal_refundable_credits. This is a missing-output failure rather than a substantive tax calculation. -us,scenario_117,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the 2-year-old child's presence as giving the head WIC categorical eligibility, but WIC categorical eligibility belongs only to pregnant, postpartum, or breastfeeding individuals, infants, and children under age 5. It also ignored the household income test: annual income of about $250,733 exceeds Arkansas WIC's 185% FPL limit for a five-person household, so the head is not WIC eligible." -us,scenario_117,payroll_tax,claude-fable-5,llm_error,other,False,"The model explicitly derived the correct $15,044.79 total from the $184,500 Social Security wage base and $3,605.79 Medicare tax, then submitted an unrelated $12,896.44 value. Its final output does not follow its own completed computation." -us,scenario_117,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model double-counted the $35,525 overtime premium even though annual gross wages already include overtime, used an obsolete Social Security cap, applied the $200,000 withholding threshold instead of the $250,000 joint Additional Medicare Tax threshold, and treated Arkansas income-tax withholding as payroll tax. It then added another invented $1,680 state withholding amount." -us,scenario_117,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model explicitly computed the correct $15,044.79 using the $184,500 Social Security wage base, but discarded that result in favor of an unsupported projected cap and submitted $14,757.34. The submitted figure also does not equal any of the alternative totals shown in its reasoning." -us,scenario_117,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model explicitly derived $11,439 of Social Security tax plus $3,605.79 of Medicare tax, totaling $15,044.79, and then submitted $13,347.55 without a supporting calculation. Its numeric output contradicts its own stated derivation." -us,scenario_117,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $176,100 as the 2026 Social Security taxable maximum instead of $184,500. That understated Social Security tax by $520.80 while its Medicare and Additional Medicare Tax treatment was correct." -us,scenario_117,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"The model substituted an estimated $178,200 Social Security wage base for the applicable $184,500 limit. This reduced Social Security tax from $11,439 to $11,048.40 and understated payroll tax by $390.60." -us,scenario_117,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $180,600 Social Security taxable maximum rather than $184,500. Its $11,197.20 Social Security calculation was therefore $241.80 too low." -us,scenario_117,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the obsolete $168,600 Social Security wage cap instead of the 2026 $184,500 cap. This understated employee Social Security tax by $985.80." -us,scenario_117,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model added the separately listed $35,525 FLSA overtime premium to $248,675 of gross wages even though the prompt states that gross annual wages already include overtime. Payroll tax must use the reported wage total once, with Social Security capped at $184,500 and Medicare applied to $248,675." -us,scenario_117,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's $14,933 answer implies a Social Security component of $11,327.21 after subtracting the correct $3,605.79 Medicare tax, equivalent to a wage cap of about $182,697 rather than $184,500. It therefore used the wrong Social Security taxable maximum." -us,scenario_117,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a $172,200 Social Security wage cap instead of $184,500. This understated Social Security tax by $762.60." -us,scenario_117,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used $11,346 as Social Security tax, corresponding to a $183,000 wage cap rather than $184,500. The cap error understated payroll tax by $93." -us,scenario_117,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"The model used a projected $183,900 Social Security taxable maximum instead of $184,500. This understated Social Security tax and total payroll tax by $37.20." -us,scenario_117,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model named the correct tax components but submitted $19,028 without applying the Social Security wage cap and Medicare calculation shown by the facts. The correct components are $11,439 of capped Social Security tax and $3,605.79 of Medicare tax, with no Additional Medicare or Arkansas employee payroll tax." -us,scenario_117,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's $22,306.24 does not result from capped Social Security tax plus Medicare tax on the reported $248,675 wages. Applying the $184,500 Social Security cap and 1.45% Medicare rate yields $15,044.79, not its unsupported total." -us,scenario_117,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model applied a rough 7.65% rate to an unspecified enlarged wage amount and even referenced nonexistent head wages. Social Security must be capped at $184,500, Medicare applies to the spouse's $248,675 wages, and the head has zero listed wages." -us,scenario_117,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly deducted the head's $8,223 health-insurance premium from the spouse's FICA wages, even though no pre-tax payroll deduction from the spouse's wages is listed. It also used a $184,000 Social Security cap instead of $184,500." -us,scenario_117,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so the required output was missing." -us,scenario_117,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model ignored the spouse's $248,675 of reported wages and assigned zero employee payroll tax. Those wages generate $11,439 of Social Security tax and $3,605.79 of Medicare tax." -us,scenario_117,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used $176,100 as the 2026 Social Security taxable maximum instead of $184,500. This understated Social Security tax by $520.80 while correctly applying Medicare tax and the joint Additional Medicare Tax threshold." +us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model included the state refund and itemized estimated Arkansas tax, medical expenses, and most of the charitable contribution. The trace instead uses AGI of $248,144.53 and deducts the $32,200 standard deduction, $25,000 overtime deduction, and $2,000 charitable deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model incorrectly asserted that credits eliminate the entire liability. The applicable deductions leave $188,944.53 taxable, generating $30,991.80 of tax before the $6,600 CTC and $24,391.80 afterward." +us,scenario_117,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the balance and an assumed rate, added an unsupported SALT deduction, and itemized them. It also included the state refund, omitted the $25,000 overtime deduction, used a $6,000 CTC instead of $6,600, and subtracted an inapplicable CDCC." +us,scenario_117,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the overtime premium as an exclusion from wages and then added further overtime deductions and credits, double-counting the preference. It also invented mortgage-interest consequences and used a $6,000 CTC rather than applying the traced $59,200 deductions and $6,600 CTC once." +us,scenario_117,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly treated the Additional Child Tax Credit formula as producing a refund even though the household has enough federal income tax liability to use its CTC nonrefundably. It also double-counted the $35,525 FLSA overtime premium despite the prompt stating that annual gross wages already include overtime, then falsely described income of $284,200 as exceeding the $400,000 joint-filer CTC phaseout threshold and invented a $3,750 refundable limit." +us,scenario_117,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for federal_refundable_credits, violating the required output contract." +us,scenario_117,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented a household-level “young-child context” pathway and transferred the two-year-old child's WIC category to the 44-year-old head. The head has no listed pregnancy, postpartum, or breastfeeding status and is neither an infant nor a child under five; independently, household income exceeds the 185% poverty limit." +us,scenario_117,payroll_tax,claude-fable-5,llm_error,other,False,"The model explicitly derived the exact $15,044.79 total using the $184,500 Social Security wage base and $248,675 Medicare wage base, then submitted an unrelated $12,896.44. Its final value does not follow any of its displayed computations." +us,scenario_117,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model double-counted the $35,525 FLSA overtime premium even though gross wages already include overtime, then incorrectly added Arkansas income-tax withholding and an invented $1,680 state payroll charge. Payroll tax here contains capped Social Security and Medicare on $248,675, with no Arkansas employee payroll tax or Additional Medicare Tax." +us,scenario_117,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model explicitly computed $11,439 of Social Security tax using the $184,500 wage base and $3,605.79 of Medicare tax, which totals $15,044.79, but submitted $14,757.34 after substituting unsupported alternative wage-base estimates. The submitted value contradicts its correct derivation." +us,scenario_117,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model correctly derived $11,439 of Social Security tax plus $3,605.79 of Medicare tax, totaling $15,044.79, and then submitted $13,347.55. That final number is disconnected from every computation in its explanation." +us,scenario_117,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $176,100 as the 2026 Social Security wage base instead of $184,500. This understated employee Social Security tax by $520.80 while its Medicare and Additional Medicare Tax treatment was correct." +us,scenario_117,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used an estimated $178,200 Social Security wage base instead of the applicable $184,500 base. That reduced Social Security tax from $11,439 to $11,048.40 and understated total payroll tax by $390.60." +us,scenario_117,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the 6.2% Social Security rate to an incorrect $180,600 wage base rather than $184,500. Its Medicare calculation was correct, but Social Security tax was understated by $241.80." +us,scenario_117,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the obsolete $168,600 Social Security wage cap instead of the 2026 cap of $184,500. This understated Social Security tax by $985.80; Medicare tax and the zero Additional Medicare Tax were otherwise computed correctly." +us,scenario_117,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model added the separately listed $35,525 FLSA overtime premium to gross wages even though the prompt states that annual gross wages already include overtime. The correct bases are $184,500 for capped Social Security and $248,675 for Medicare, yielding $15,044.79." +us,scenario_117,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's $14,933 result implies that it used a Social Security cap below $184,500 while applying Medicare to the stated wages. The correct capped Social Security amount is $11,439, which combines with $3,605.79 of Medicare tax." +us,scenario_117,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used $172,200 as the Social Security wage cap instead of $184,500. This understated Social Security tax by $762.60 while correctly applying Medicare to $248,675 and excluding Additional Medicare Tax." +us,scenario_117,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used $11,346 as capped Social Security tax, corresponding to a wage base of $183,000 rather than $184,500. Social Security tax is $11,439, so the model understated the total by $93." +us,scenario_117,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"The model used a projected $183,900 Social Security wage base instead of $184,500. That understated Social Security tax by $37.20; its Medicare and Additional Medicare Tax calculations were correct." +us,scenario_117,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model stated the correct components but submitted $19,028 without computing them from their proper bases. Capped Social Security is $11,439 and Medicare on $248,675 is $3,605.79, with no Additional Medicare Tax or Arkansas employee payroll tax." +us,scenario_117,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The submitted $22,306.24 cannot result from 6.2% Social Security capped at $184,500 plus 1.45% Medicare on $248,675. Those stated components produce $15,044.79, so the model applied its payroll-tax rates to an inflated or uncapped base." +us,scenario_117,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model applied a rough 7.65% rate to unspecified high wages and entertained unlisted head wages, despite the instruction that unlisted wages are zero. It also failed to cap the 6.2% Social Security component at $184,500 while applying Medicare only to the spouse's $248,675 wages." +us,scenario_117,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model deducted the head's $8,223 insurance premium from the spouse's FICA wages, even though it was not identified as the spouse's pretax payroll deduction. It also used a $184,000 Social Security wage base instead of $184,500; Medicare applies to the full $248,675 stated wage amount." +us,scenario_117,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no payroll-tax value or explanation. It therefore failed the required structured-output contract rather than completing a substantive calculation. +us,scenario_117,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model ignored the spouse's explicitly listed $248,675 of wages and assigned no employee payroll tax. Those wages generate $11,439 of capped Social Security tax and $3,605.79 of Medicare tax." +us,scenario_117,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used an estimated $176,100 Social Security wage base instead of $184,500. This understated Social Security tax by $520.80 while its Medicare, Additional Medicare Tax, and Arkansas treatment was correct." +us,scenario_117,payroll_tax,qwen3.8-max,llm_error,other,False,"The model explicitly calculated $11,439 of Social Security tax, approximately $3,606 of Medicare tax, and zero Additional Medicare Tax, which yields about $15,045, but submitted $20,603. Its final value contradicts its own component calculation." us,scenario_117,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_117,spouse_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated household association with a young child as sufficient for spouse WIC eligibility and skipped the WIC income test. PolicyEngine applies the 185% FPL income limit to the household, and this household's roughly $250,733 income is above that threshold, so the spouse is not WIC eligible." -us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used the Arkansas standard deduction and taxable income near $245,805 instead of applying the $24,659.28 itemized deduction that reduces taxable income to $223,485.25. It also omitted the $60 qualified-individual credits, subtracting only the $145 personal credits." -us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted the separately listed $35,525 FLSA overtime premium on top of gross wages and invented deductions for health premiums, employee expenses, and self-employment tax. It then used the wrong Arkansas taxable income, obsolete 5% top rate, and an unsupported $1,050 credit total instead of $223,485.25, the applicable brackets, and $205 of credits." -us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented roughly $22,500 of mortgage interest from the mortgage balance and combined it with the full charitable contribution to estimate $48,400 of itemized deductions, rather than using the traced $24,659.28 deduction. Its final $11,264 answer also contradicts its own approximate $7,645 calculation and never applies the $60 qualified-individual credits." -us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model included the $2,589 state tax refund in Arkansas income and invented mortgage interest to reduce taxable income to about $202,000. It then produced $11,820 despite a 3.9% top rate being incapable of generating that liability on its stated tax base, and it omitted the $60 qualified-individual credits." -us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model overstated itemized deductions by including an unsupported mortgage-interest amount and medical deductions, reducing taxable income to about $190,000 instead of $223,485.25. It also subtracted only $145 of personal credits and omitted the additional $60 credit." -us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the separately reported FLSA overtime premium from wages, lowering Arkansas AGI to $215,209 instead of $248,144.53, and then constructed its own $36,348 itemized deduction. It also applied an unsupported $240 Arkansas child-care credit rather than the traced $60 qualified-individual credits." -us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model invented mortgage interest from the outstanding mortgage balance and therefore used taxable income near $202,205 instead of $223,485.25. Its $12,500 output contradicts its own bracket estimate near $8,086 and omits the traced $205 credit computation." -us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model took a $5,000 standard deduction rather than the $24,659.28 itemized deduction, leaving taxable income at $243,145 instead of $223,485.25. It also failed to subtract the $205 of nonrefundable Arkansas credits." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model started from $250,734 rather than the traced $248,144.53 AGI and used $26,641 of itemized deductions rather than $24,659.28. It subtracted only the $145 personal credits and omitted the $60 qualified-individual credits." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $10,560 estimate does not follow the Arkansas computation: $223,485.25 of taxable income produces $8,189.05 under the applicable joint-filer brackets, not $10,560. The answer also fails to reflect the final $205 reduction for nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's generic invocation of Arkansas rates yields $8,768 instead of the traced $8,189.05 bracket liability on $223,485.25 of taxable income. It also does not apply the full $205 of nonrefundable credits needed to reach $7,984.05." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used approximate AGI of $250,734 rather than $248,144.53 and did not establish the traced $24,659.28 itemized deduction or $223,485.25 taxable income. Its $8,623 result therefore does not incorporate the exact bracket liability and full $205 credit reduction." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model supplied no computation supporting $8,234.12. The required sequence is $223,485.25 of taxable income, $8,189.05 of bracket tax, and a $205 nonrefundable-credit reduction to $7,984.05." -us,scenario_117,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model deducted the full $25,900 charitable contribution instead of using total traced itemized deductions of $24,659.28, understating taxable income by $1,240.25. More importantly, it applied an inapplicable bracket schedule reaching 4.7% and 4.4% and did not subtract the $205 of nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The $1,005 answer treats a high-income Arkansas household as having only modest liability without applying the joint-filer brackets to $223,485.25. Those brackets produce $8,189.05 before the $205 nonrefundable-credit reduction." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated deductions and offsets as eliminating Arkansas tax. They reduce AGI only to $223,485.25 of taxable income, which generates $8,189.05 before $205 of credits and leaves $7,984.05 due." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model deducted the full $25,900 charitable amount and ignored the traced total itemized deduction of $24,659.28, producing $222,245 rather than $223,485.25 of taxable income. It also used $160 of credits instead of the correct $205, which consists of $145 in personal credits plus $60 in qualified-individual credits." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model stated that no nonrefundable state credit applied, but this household receives $145 in personal credits and $60 in additional qualified-individual credits. Subtracting that $205 from the $8,189.05 bracket liability yields $7,984.05, not $8,800." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used an approximate charitable-deduction calculation rather than the traced $24,659.28 itemized deduction and did not identify the resulting $223,485.25 taxable income. It also failed to apply the exact $205 credit total to the $8,189.05 bracket tax." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model came close but did not apply the exact Arkansas credit computation: $145 for the two adults and three children plus $60 for qualified individuals. The exact $205 reduction from $8,189.05 produces $7,984.05, not $7,995.31." -us,scenario_117,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model rounded the Arkansas liability to $8,000 instead of performing the specified computation. Applying the brackets to $223,485.25 and subtracting $205 of nonrefundable credits produces the non-rounded annual amount of $7,984.05." -us,scenario_117,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a top marginal rate near 4.4% instead of the applicable Arkansas 2026 bracket schedule, whose liability on $223,485.25 is $8,189.05. It also failed to subtract the full $205 of personal and qualified-individual credits." -us,scenario_117,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an unexplained AGI of $242,511 and deducted the full $25,900 charitable contribution, producing taxable income of $216,611 rather than $223,485.25. It treated the tax as a flat 3.9% amount and omitted the $205 of nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not be evaluated as a substantive tax calculation." -us,scenario_117,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the joint standard deduction and taxable income near $245,654 instead of the $24,659.28 itemized deduction and $223,485.25 taxable income. Although it subtracted the $145 personal credits, it omitted the additional $60 qualified-individual credits." -us,scenario_117,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly set Arkansas liability to zero. The household retains $223,485.25 of taxable income after itemized deductions, producing $8,189.05 of tax before the $205 credit reduction." -us,scenario_117,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model double-counted the $35,525 FLSA overtime premium by adding it to the annual gross-wage total, inflating income to $286,259. It then abandoned its itemized-deduction calculation, used the standard deduction and an obsolete flat 4.4% rate, and omitted the $60 qualified-individual credits." +us,scenario_117,spouse_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly transferred a young child's WIC categorical status to the spouse. Living with a WIC-age child does not make an adult eligible; the spouse has no stated pregnancy, postpartum, or breastfeeding status and the household also fails WIC's income test." +us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used an approximately $2,340 standard deduction instead of the $24,659.28 Arkansas itemized deduction, leaving taxable income far above $223,485.25. It also failed to subtract the $60 additional qualified-individual credits alongside the $145 personal/dependent credits." +us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted the separately listed FLSA overtime premium on top of gross wages, treated expenses as AGI adjustments without a valid pathway, and invented self-employment tax despite no self-employment income. It then used the wrong Arkansas rates and $1,050 of unsupported credits instead of $205." +us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model imputed roughly $22,500 of mortgage interest from the mortgage balance and thereby overstated itemized deductions; the traced Arkansas itemized deduction is $24,659.28 in total. Its final $11,264 also contradicts its own intermediate estimate near $7,645 and does not follow from the stated calculation." +us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model included the $2,589 state tax refund in Arkansas income and imputed mortgage interest, producing the wrong AGI and deduction base. It then applied the 3.9% schedule incorrectly because tax on its claimed $202,000 taxable income cannot produce $11,820." +us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to about $190,000 by overstating itemized deductions, including an unsupported mortgage-interest estimate. Arkansas taxable income is $223,485.25, and it also omitted the $60 additional qualified-individual credits." +us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $35,525 FLSA overtime premium from Arkansas income even though it is already included in the listed annual gross wages and no traced Arkansas subtraction applies. It also constructed unsupported SALT, employee-expense, and Arkansas CDCC deductions or credits rather than using $24,659.28 of itemized deductions and $205 of traced nonrefundable credits." +us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model imputed about $20,000 of mortgage interest from the loan balance instead of using the traced $24,659.28 total itemized deduction. Its $12,500 answer is also arithmetically disconnected from its own calculation of roughly $8,086." +us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model took a $5,000 standard deduction rather than the $24,659.28 itemized deduction, leaving taxable income at $243,145 instead of $223,485.25. It also did not subtract the $205 of nonrefundable credits from bracket tax." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used AGI of $250,734 rather than $248,144.53 and itemized deductions of $26,641 rather than $24,659.28, producing the wrong taxable income. It subtracted only $145 of credits and omitted the $60 additional qualified-individual credits." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer does not implement the traced Arkansas computation: $223,485.25 of taxable income produces $8,189.05 of bracket tax, followed by $205 of nonrefundable credits. The $10,560 estimate reflects an incorrect tax base, rate schedule, or both." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model supplied only a generic appeal to Arkansas rates and did not apply the traced schedule to $223,485.25 of taxable income. That schedule yields $8,189.05 before subtracting $205 of nonrefundable credits, not $8,768." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model started from approximately $250,734 rather than Arkansas AGI of $248,144.53 and did not establish the traced $24,659.28 itemized deduction. Those errors prevent the correct $223,485.25 taxable-income base and $7,984.05 post-credit result." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model gave no computation supporting $8,234.12. The traced bracket tax is $8,189.05 and subtracting the full $205 of nonrefundable credits yields $7,984.05." +us,scenario_117,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Although the model was close to the correct income base, it used obsolete and internally inconsistent Arkansas brackets extending to 4.7% and 4.4%. It also failed to subtract the traced $205 of nonrefundable credits from the correct $8,189.05 bracket liability." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated a high-income Arkansas household's liability as modest without applying the bracket schedule to $223,485.25 of taxable income. The schedule produces $8,189.05 before the $205 credit subtraction, so $1,005 omits most of the tax liability." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated deductions and offsets as eliminating the liability, but the traced deductions reduce AGI only to $223,485.25 of taxable income. That amount generates $8,189.05 of tax, and the $205 of credits do not reduce it to zero." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model deducted the full $25,900 charitable contribution and stopped at $222,245 instead of using the traced total Arkansas itemized deduction of $24,659.28 and taxable income of $223,485.25. It also used $160 of credits rather than the traced $205." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model asserted that no nonrefundable state credit applied, omitting both the $145 personal/dependent credits and $60 additional qualified-individual credits. The full $205 must be subtracted from the $8,189.05 bracket liability." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model did not reproduce the exact Arkansas deduction base or credit total. Applying $24,659.28 of itemized deductions gives $223,485.25 of taxable income, and subtracting all $205 of credits from $8,189.05 yields $7,984.05 rather than $8,107." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"The model rounded through an undisclosed approximation instead of completing the traced calculation. The exact bracket liability of $8,189.05 less exactly $205 in nonrefundable credits is $7,984.05, not $7,995.31." +us,scenario_117,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model rounded the Arkansas liability to $8,000 instead of calculating the requested amount. The exact computation is $8,189.05 of bracket tax minus $205 of nonrefundable credits, or $7,984.05." +us,scenario_117,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a top marginal rate near 4.4% rather than the applicable Arkansas bracket schedule. Applying the correct schedule to $223,485.25 yields $8,189.05 before the $205 credit subtraction." +us,scenario_117,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model started from an unexplained AGI of $242,511, deducted the full charitable contribution, and treated 3.9% as a flat rate. The traced computation instead uses $248,144.53 of AGI, $24,659.28 of itemized deductions, graduated bracket tax of $8,189.05, and $205 of credits." +us,scenario_117,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model approximated the taxable base and subtracted only unspecified small personal credits while asserting no major nonrefundable credits. It omitted the traced $60 additional qualified-individual credits and failed to compute the exact $8,189.05 bracket liability on $223,485.25." +us,scenario_117,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric output for the requested variable. This is a missing-output contract failure rather than a completed Arkansas tax computation. +us,scenario_117,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the joint standard deduction and taxable income near $245,654 instead of the larger $24,659.28 itemized deduction and $223,485.25 taxable income. It also subtracted only $145 and omitted the $60 additional qualified-individual credits." +us,scenario_117,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model asserted zero liability despite $223,485.25 of Arkansas taxable income. The bracket calculation produces $8,189.05, and the available $205 of nonrefundable credits leave $7,984.05 rather than eliminating the tax." +us,scenario_117,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted the $35,525 overtime premium on top of gross wages, included the state refund, and ultimately discarded its own itemized-deduction calculation in favor of the standard deduction. It also used a 4.4% flat rate instead of the applicable graduated schedule and omitted the $60 additional qualified-individual credits." +us,scenario_117,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented a $28,867 married-filing-jointly deduction and began from an unsupported $213,150 income figure rather than the traced AGI and itemized deduction. It also subtracted only a $60 dependent credit instead of the full $205 credit total, and its submitted $5,105 does not match its own stated $7,127 calculation." us,scenario_117,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_118,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the New York SSI-recipient Medicaid category and treated disability as requiring unspecified additional Medicaid criteria that the facts did not prove. The correct computation first establishes SSI recipient status from the aged criterion plus the SSI resource and income tests, then grants automatic Medicaid eligibility in New York." -us,scenario_118,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that Social Security retirement benefits are not subject to employee payroll tax, then nevertheless applied payroll tax to the $2,800 Social Security amount. Its $172.90 answer is consistent with taxing non-wage retirement income at roughly the employee Social Security and Medicare rate instead of using the zero covered-wage payroll tax base." -us,scenario_118,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model counted only $233.33 of monthly Social Security and omitted the modeled $780.67 SSI benefit from SNAP gross income. It then used an unsupported utility allowance to drive net income to zero, while the applicable calculation yields $194.38 of net income and a $58.20 expected contribution." -us,scenario_118,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model declared this extremely low-income elderly household ineligible despite its SSI and TANF non-cash categorical-eligibility pathways and its satisfaction of the net-income, gross-income, and asset tests. Those tests produce a positive monthly allotment, not zero." -us,scenario_118,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the $780.67 monthly modeled SSI benefit from gross income and treated the listed ownership costs as an uncapped deduction that eliminated all net income. SNAP net income is $194.38, so the household owes a $58.20 expected contribution rather than receiving the maximum allotment." -us,scenario_118,snap,claude-opus-4.8,llm_error,other,False,"The model’s stated arithmetic supports roughly $3,400 annually, but it submitted $2,533 without any computation producing that figure. It also began from Social Security alone instead of the $1,014 monthly SNAP gross income that includes modeled SSI." -us,scenario_118,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly stated a $298 maximum and about $29 of net income, but then subtracted roughly $95 instead of 30% of $29 to obtain $203. The applicable calculation uses $194.38 of net income and a $58.20 contribution, yielding about $239.80 for most months." -us,scenario_118,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model explicitly assumed there was no SSI and therefore started with only $233.33 of monthly income. PolicyEngine first computes $780.67 of SSI, counts it in SNAP gross income, and obtains $194.38 of net income rather than zero." -us,scenario_118,snap,claude-sonnet-5,llm_error,other,False,"The model reasoned that net income was at or below zero and that the maximum was about $3,504 annually, then imposed an unexplained reduction to $2,555. The actual reduction is exactly 30% of $194.38 monthly net income, with the resulting monthly allotments summed across the year." -us,scenario_118,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model omitted modeled SSI from countable gross income and used the shelter deduction to reduce net income to zero. Counting $780.67 of SSI with Social Security produces $1,014 gross and $194.38 net income, so the benefit is below the maximum." -us,scenario_118,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included an estimated SSI amount but applied a standard utility allowance and shelter deduction that reduced net income to zero. The traced deductions leave $194.38 of SNAP net income, producing a $58.20 expected contribution rather than the maximum allotment." -us,scenario_118,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model treated the supplied facts as incapable of generating a benefit even though SSI and TANF non-cash categorical eligibility apply and all income and asset tests pass. The SNAP formula therefore produces a positive annual benefit. -us,scenario_118,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model estimated SSI incorrectly and stopped the deduction sequence at $679 of net income after subtracting only standard and excess-shelter deductions. The traced SNAP calculation yields $1,014 gross income and $194.38 net income, reducing the $298 maximum by only $58.20." -us,scenario_118,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model replaced the ordinary net-income benefit formula with a fixed NYSNIP maximum of $291 per month. PolicyEngine applies a $298 maximum and subtracts 30% of $194.38 net income, with updated parameters in part of the year." -us,scenario_118,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted $3,492 is a constant $291 monthly maximum and does not subtract the household’s expected contribution. The calculation uses $194.38 of net income, a $58.20 contribution, and different parameter values across the year." -us,scenario_118,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model counted only Social Security, omitted $780.67 of modeled SSI, and consequently reduced net income to zero with shelter costs. Countable gross income is $1,014 monthly and net income is $194.38, so SNAP is not the maximum allotment." -us,scenario_118,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model asserted that no expenses or eligibility facts were listed despite the household’s age, disability, $70 assets, mortgage interest, property tax, and modeled SSI. SSI and TANF non-cash categorical eligibility apply, and all three financial tests yield eligibility and a positive allotment." -us,scenario_118,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required additional supporting facts and returned zero. The supplied facts generate modeled SSI, categorical eligibility, and passing gross-income, net-income, and asset tests, which are sufficient for a positive SNAP amount." -us,scenario_118,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model omitted modeled SSI from SNAP gross income and invoked a $20 SSI/SNAP exclusion plus the standard deduction to set net income to zero. The traced calculation counts $1,014 of gross income and leaves $194.38 net, while the maximum allotment is $298 rather than $243 for most months." -us,scenario_118,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model’s answer is consistent with treating the household as nearly entitled to a constant maximum based only on $2,800 of Social Security. SNAP also counts modeled SSI, leaving $194.38 net income and requiring a $58.20 monthly contribution." -us,scenario_118,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,The model combined Social Security and SSI but produced an unsupported $115 monthly benefit from its estimated deductions. The traced deduction sequence yields $194.38 net income and a benefit near $239.80 in most months. -us,scenario_118,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model’s $123 monthly allotment implies substantially overstated countable net income after including SSI. The correct deduction sequence leaves $194.38 net income, so only $58.20 is subtracted from the $298 maximum in most months." -us,scenario_118,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model returned zero on the false premise that SNAP calculation inputs were absent. The household facts generate SSI, categorical eligibility, and passing income and asset tests, followed by a positive allotment under the 30% contribution formula." -us,scenario_118,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model correctly recognized SSI categorical eligibility but overstated net income at about $735 despite describing unlimited excess-shelter deductions. The traced calculation yields $194.38 net income and a $58.20 expected contribution, not an $81 monthly benefit." -us,scenario_118,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model omitted modeled SSI and incorrectly treated shelter costs as zero even though mortgage interest and real estate taxes were explicitly listed. The traced computation starts from $1,014 monthly gross income and ends at $194.38 net income." -us,scenario_118,snap,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model counted only Social Security and let the shelter deduction eliminate net income, then used an overstated $309 maximum. Modeled SSI raises gross income to $1,014, deductions leave $194.38 net, and the applicable maximum is $298 for most months." -us,scenario_118,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,The model included roughly the right combined Social Security and SSI income but ignored the explicitly listed mortgage-interest and property-tax shelter costs because they were not labeled as a separate SNAP input. Applying the traced deductions reduces net income from its $892 figure to $194.38 and raises the monthly benefit from $30 to about $239.80. -us,scenario_118,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model inverted the SNAP income logic by saying the household’s income was too low to qualify and then claiming low net income generated zero benefit. Low net income increases SNAP, and this categorically eligible household receives the maximum minus a 30% contribution." -us,scenario_118,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model counted only $233.33 of Social Security, omitted modeled SSI, and failed to apply the listed shelter costs in its final net-income calculation. The correct calculation uses $1,014 gross income, $194.38 net income, and a $58.20 expected contribution." -us,scenario_118,ssi,claude-fable-5,llm_error,other,False,"The model explicitly derived the correct $780.67 monthly benefit and $9,368 annual benefit, then submitted the unrelated $6,144 figure. Its final value contradicts its own completed SSI calculation." -us,scenario_118,ssi,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model compared $2,800 of annual Social Security income with a monthly SSI maximum and incorrectly declared the person income-ineligible. Converting income to $233.33 per month and applying the $20 monthly exclusion leaves $213.33 countable, producing $780.67 monthly SSI." -us,scenario_118,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a $967 federal rate instead of the 2026 $994 monthly maximum and added an unsupported New York supplement. This output is the federal SSI calculation: $994 minus $213.33 monthly countable income, annualized to $9,368." -us,scenario_118,ssi,claude-opus-4.8,llm_error,other,False,"The model correctly concluded that categorical and resource eligibility were satisfied and that the income calculation produced a positive SSI payment, but submitted zero. Its output directly contradicts its stated positive-benefit derivation." -us,scenario_118,ssi,claude-opus-5,llm_error,thresholds_rates,False,"The model used an unexplained $667 monthly result after correctly identifying $213.33 of monthly countable income. Applying that income to the $994 monthly maximum yields $780.67, not $667, and no New York adjustment reduces it to $8,004." -us,scenario_118,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the $967 monthly rate instead of the 2026 $994 rate. With the correctly identified $213.33 monthly countable income, the federal SSI payment is $780.67 per month and $9,368 annually." -us,scenario_118,ssi,claude-sonnet-5,llm_error,other,False,"The model's own stated arithmetic gives $12,152 minus $2,780 equal to $9,372, yet it submitted $6,212 after an unsupported adjustment. The trace instead uses the $11,928 annual maximum and $2,560 countable income, yielding $9,368." -us,scenario_118,ssi,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a projected $991 federal rate and added a $54 New York supplement. The applicable maximum is $994 per month with no state supplement in this output, so subtracting $213.33 and annualizing gives $9,368." -us,scenario_118,ssi,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model added an $87 monthly New York supplement and used a projected $991 federal rate. The SSI output uses the $994 monthly maximum alone, reduced by $213.33 of monthly countable income." -us,scenario_118,ssi,gemini-3.1-flash-lite-preview,llm_error,period_annualization,False,"The model treated $2,800 of annual Social Security income as exceeding the monthly SSI maximum. The income is $233.33 per month, and after the $20 exclusion only $213.33 reduces the $994 monthly benefit." -us,scenario_118,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used an $11,316 federal maximum and added a $1,044 New York supplement. The applicable annual maximum is $11,928 without that supplement, and subtracting the correctly calculated $2,560 countable income yields $9,368." -us,scenario_118,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model substituted the 2024 $943 monthly federal rate and added an $87 monthly New York supplement. The 2026 SSI output uses $994 per month without that supplement, producing $9,368 after the income reduction." -us,scenario_118,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly calculated $2,560 of annual countable income but used an implied annual maximum of $11,796. The 2026 maximum is $11,928, so the benefit is $11,928 minus $2,560, or $9,368." -us,scenario_118,ssi,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $991 monthly federal rate and added a $94 monthly New York supplement. The applicable maximum is $994 per month with no supplement included, yielding $9,368 annually after subtracting countable income." -us,scenario_118,ssi,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model ignored the explicit age-74, blind, disabled, $70-resource, and $2,800-income facts that establish and quantify SSI eligibility. The aged pathway and resource test are satisfied, and the income reduction leaves a positive $9,368 annual payment." -us,scenario_118,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly claimed that qualifying age, disability, income, and resource facts were absent even though all are listed. Age 74 establishes the aged category, $70 passes the resource test, and countable income reduces rather than eliminates the benefit." -us,scenario_118,ssi,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly calculated $2,560 of annual countable income but used an invented $982.67 monthly federal rate. Using the $994 monthly rate gives an $11,928 maximum and a $9,368 benefit." -us,scenario_118,ssi,gpt-5.6-terra,llm_error,period_annualization,False,"The model applied the $20 general exclusion only once for the year, subtracting $20 from $2,800 to get $2,780. The exclusion applies monthly, totaling $240 annually, so countable income is $2,560 and SSI is $9,368." -us,scenario_118,ssi,grok-4.3,llm_error,categorical_eligibility,False,"The model disregarded the supplied SSI inputs: age 74, blindness, disability, $70 of bank assets, and $2,800 of Social Security income. Those facts establish eligibility and produce $9,368 after the monthly general exclusion and income reduction." -us,scenario_118,ssi,grok-4.5,llm_error,thresholds_rates,False,"The model used a projected $991 federal rate and added an $87 New York supplement. The output uses the $994 monthly SSI maximum alone, reduced by $213.33 per month, for $9,368 annually." -us,scenario_118,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"The model used the 2024 $943 federal rate as a proxy and added an $87 New York supplement. The requested 2026 calculation uses the $994 monthly maximum without that supplement, yielding $9,368." -us,scenario_118,ssi,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly calculated $2,560 of countable annual income but used a $991 monthly maximum. The 2026 monthly maximum is $994, making the annual maximum $11,928 and the benefit $9,368." -us,scenario_118,ssi,kimi-k3,llm_error,thresholds_rates,False,"The model correctly derived $9,368 of federal SSI, then incorrectly added a $1,044 New York living-alone supplement. That supplement is not part of this SSI output, so the correct submitted amount is the already-derived $9,368." -us,scenario_118,ssi,minimax-m3,llm_error,thresholds_rates,False,"The model stated the impossible comparison that $233 of monthly Social Security income exceeds a $967 monthly SSI rate. After the $20 exclusion, $213.33 is below the applicable $994 maximum and reduces the benefit to $780.67 per month." -us,scenario_118,ssi,qwen-3.7-max,llm_error,other,False,"The model derived a positive monthly and annual SSI payment but submitted zero. Using the correct 2026 $994 monthly maximum instead of $943 raises its positive derivation to $780.67 per month and $9,368 annually." -us,scenario_118,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly denied the homeowner real property tax credit after focusing on the absence of rent and earned-income or child credits. The head's age of 74 and $1,633.50 of real estate taxes qualify the household for the $375 refundable New York credit." -us,scenario_118,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,The model omitted New York's refundable real property tax credit and incorrectly treated the state's refundable-credit landscape as limited to credits such as the EITC. The senior homeowner facts generate a $375 real property tax credit despite zero New York taxable income. -us,scenario_118,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,The model identified the senior real property tax credit but replaced its applicable $375 benefit with an unsupported approximate formula yielding $63. The household reaches the $375 credit based on the head's age and qualifying real estate taxes. -us,scenario_118,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model incorrectly made refundable-credit eligibility depend on positive taxable income or tax liability. New York's real property tax credit is refundable, and this 74-year-old homeowner receives $375 even though the Social Security income produces no New York taxable income." -us,scenario_118,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model treated the absence of rent and positive New York taxable income as disqualifying, overlooking the separate homeowner pathway based on real estate taxes. The head's age of 74 and $1,633.50 property-tax payment produce the $375 refundable credit." -us,scenario_118,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,The model correctly identified the real property tax credit but imposed the $75 cap for a non-senior single claimant. The age-65-or-older pathway applies to the 74-year-old head and carries the $375 credit used here. -us,scenario_118,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model wrongly dismissed the homeowner circuit-breaker credit because no rent was paid and described the senior benefit as negligible. The reported real estate taxes activate the homeowner pathway, and the qualifying senior receives $375." -us,scenario_118,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly included the $375 real property tax credit but incorrectly added a $75 New York household credit to refundable credits. The household credit does not contribute to this refundable-credit output, so only the $375 real property tax credit belongs in the total." -us,scenario_118,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model's blanket ineligibility conclusion omitted the New York real property tax credit. The head is 74 and paid $1,633.50 in real estate taxes, producing a refundable credit of $375." -us,scenario_118,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,The model considered only earned-income and child-based credits and omitted New York's senior homeowner real property tax credit. That separate pathway yields $375 from the head's age and property taxes. -us,scenario_118,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model failed to apply New York's refundable real property tax credit. The 74-year-old New York homeowner's $1,633.50 real estate-tax payment generates $375." -us,scenario_118,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model used an inapplicable 50%-of-excess formula and an unsupported $1,000 senior maximum to calculate $768. The applicable New York real property tax credit computation caps this household's benefit at $375." -us,scenario_118,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,The answer implies that the model did not recognize age and property taxes as triggers for New York's refundable real property tax credit. Those listed facts produce a $375 credit. -us,scenario_118,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model overlooked the specifically indicated senior homeowner credit: age 74 and $1,633.50 of real estate taxes are qualifying inputs for New York's refundable real property tax credit. The resulting amount is $375." -us,scenario_118,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,The model incorrectly inferred that Social Security-only income precludes all New York refundable credits. The real property tax credit follows a senior homeowner pathway and yields $375 independently of positive taxable earnings. -us,scenario_118,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model substituted a $90 New York household credit for the applicable refundable real property tax credit. This output contains the $375 senior homeowner credit, not the household credit selected from filing status and AGI." -us,scenario_118,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The model's blanket statement that no New York credits qualify omits the real property tax credit. The 74-year-old homeowner's reported real estate taxes generate $375. -us,scenario_118,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model restricted refundable New York credits to the EITC and child-related programs. It omitted the separate refundable real property tax credit, which pays this senior homeowner $375." -us,scenario_118,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model catalogued earned-income, child, tuition, and childcare credits but failed to evaluate the listed age and real estate taxes under New York's real property tax credit. That homeowner credit contributes $375." -us,scenario_118,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model incorrectly required New York tax liability or children for a refundable credit. The senior homeowner real property tax credit is payable despite zero income-tax liability and equals $375 here. -us,scenario_118,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model focused on zero taxable income and the absence of earned-income or child-credit eligibility, omitting the real property tax credit. The head's age and reported property taxes satisfy that separate refundable pathway and yield $375." -us,scenario_119,child1_chip_eligible,gemini-3.5-flash,llm_error,health_coverage,False,"The model applied only the under-19 and Virginia FAMIS income tests. It ignored Child 1’s employer-sponsored insurance, which prevents the child from satisfying CHIP’s coverage-related categorical requirements." -us,scenario_119,child1_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,The model treated age and household income as sufficient for CHIP eligibility. It omitted the coverage criterion and therefore failed to account for Child 1’s employer-sponsored insurance. -us,scenario_119,child1_chip_eligible,gpt-5.5,llm_error,health_coverage,False,"The model inferred CHIP eligibility solely from being above the Medicaid child limit and within the CHIP income range. It failed to apply CHIP’s separate coverage-related categorical test to Child 1, who has employer-sponsored insurance." -us,scenario_119,child1_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,The model stopped after applying the under-19 and estimated-income-range rules. It ignored Child 1’s employer-sponsored insurance and thus did not apply CHIP’s coverage-related categorical requirement. -us,scenario_119,child1_chip_eligible,grok-4.5,llm_error,health_coverage,False,The model used its estimated 192% FPL MAGI as dispositive once it placed the child below the FAMIS ceiling. Income eligibility alone is insufficient because Child 1 has employer-sponsored insurance and fails CHIP’s coverage-related categorical criteria. -us,scenario_119,child1_chip_eligible,grok-build-0.1,llm_error,health_coverage,False,"The model focused on a calculated MAGI of $46,180 and the asserted 143–200% FPL CHIP band. It never applied the separate coverage criterion to Child 1’s employer-sponsored insurance." -us,scenario_119,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer for child1_chip_eligible. The required numeric output was therefore missing. -us,scenario_119,child1_chip_eligible,kimi-k3,llm_error,health_coverage,False,The model correctly distinguished potential eligibility from enrollment but treated age and its MAGI estimate as the complete potential-eligibility test. It omitted CHIP’s coverage-related categorical requirement despite Child 1’s employer-sponsored insurance. -us,scenario_119,child1_medicaid_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model correctly found income above Virginia's child Medicaid limit, then incorrectly treated FAMIS/CHIP-funded coverage up to roughly 205% FPL as a Medicaid eligibility pathway. FAMIS/CHIP eligibility does not change child1's Medicaid category from NONE." -us,scenario_119,child1_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used a family-of-four poverty guideline even though the listed household contains only the head and two children, and it merged the Medicaid and FAMIS/CHIP thresholds into one test. The engine's MAGI calculation is 1.92 times FPL, above the Medicaid limit, so child1 has no Medicaid eligibility category." -us,scenario_119,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that 1.92-times-FPL household MAGI was within Virginia's child Medicaid limit. For a 14-year-old, that income exceeds the applicable Medicaid threshold, and no other Medicaid pathway applies." +us,scenario_118,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model stopped at a generic disability-based Medicaid inquiry and failed to apply New York's automatic SSI-recipient pathway. The head meets SSI's aged criterion and passes the $70 resource and $213.33 countable-income tests, producing a $780.67 monthly SSI benefit and automatic Medicaid eligibility." +us,scenario_118,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that the $2,800 Social Security retirement benefit is not subject to employee payroll tax and that no wages were provided, but then contradicted that reasoning by assigning $172.90 of employee-side tax. It improperly placed retirement benefits or another nonexistent amount in the payroll-tax base instead of applying the tax rates to a zero earned-income base." +us,scenario_118,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model reduced SNAP net income to zero by treating the listed mortgage interest, property tax, and an invented utility allowance as an uncapped excess-shelter deduction. PolicyEngine’s deductions leave $194.38 of monthly net income, so the household owes a $58.20 expected contribution instead of receiving the maximum allotment." +us,scenario_118,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model concluded that extremely low income disqualified the household, reversing SNAP’s income test and ignoring categorical eligibility through modeled SSI receipt and TANF non-cash eligibility. The household passes the net-income, gross-income, and asset tests and receives a positive allotment." +us,scenario_118,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model deducted all listed mortgage interest and property tax as SNAP shelter expense and drove net income to zero. PolicyEngine’s SNAP deduction calculation leaves $194.38 of net income, producing a 30% expected contribution rather than the maximum benefit." +us,scenario_118,snap,claude-opus-4.8,llm_error,other,False,"The reasoning first derived roughly $3,400 from its own $29 monthly net-income estimate, then submitted $2,533 without a corresponding computation. It also omitted PolicyEngine’s modeled SSI income and the actual $194.38 SNAP net-income result used in the benefit formula." +us,scenario_118,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model states that $298 minus 30% of $29 equals about $203, but that arithmetic equals about $289. It also omitted modeled SSI from the SNAP income calculation; PolicyEngine uses $194.38 of net income and a $58.20 expected contribution." +us,scenario_118,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated the listed ownership costs as a shelter deduction large enough to eliminate all net income and explicitly asserted that no SSI was present. PolicyEngine models $780.67 of SSI receipt and leaves $194.38 of SNAP net income after applicable deductions, so the benefit is below the maximum." +us,scenario_118,snap,claude-sonnet-5,llm_error,other,False,"The model claimed net income was at or below zero and that the household received the maximum allotment, then reduced the annual answer to $2,555 without calculating any expected contribution. PolicyEngine instead calculates $194.38 of monthly net income and applies the 30% reduction formula month by month." +us,scenario_118,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model drove net income to zero through standard, medical, and shelter deductions and therefore awarded the maximum allotment. PolicyEngine’s deduction sequence leaves $194.38 of net income, generating a $58.20 expected contribution." +us,scenario_118,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included modeled SSI but used the wrong SSI amount, invented a standard utility allowance, and deducted shelter costs until net income reached zero. PolicyEngine uses $780.67 of SSI receipt and calculates $194.38 of SNAP net income rather than zero." +us,scenario_118,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated low income and expenses as insufficient to generate a SNAP calculation. The household passes every eligibility test, including categorical eligibility through SSI receipt and TANF non-cash eligibility, and its positive allotment follows from $194.38 of net income." +us,scenario_118,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an invented $817 monthly SSI amount and an incomplete deduction calculation to produce $679 of net income. PolicyEngine models $780.67 of SSI and derives $194.38 of SNAP net income, which yields a much smaller expected contribution." +us,scenario_118,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model substituted an NYSNIP maximum-benefit shortcut for PolicyEngine’s ordinary monthly allotment calculation. SSI supports categorical eligibility, but the benefit still equals the maximum allotment minus 30% of $194.38 net income and is not automatically the maximum." +us,scenario_118,snap,gemini-3.6-flash,llm_error,other,False,"The model asserted that PolicyEngine produced $3,492 without showing the net-income or expected-contribution computation. The trace instead produces $194.38 of net income, a $58.20 expected contribution, and monthly allotments that sum to $2,903.94." +us,scenario_118,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model applied the full mortgage-interest and property-tax amount as an uncapped shelter deduction and reduced net income to zero. PolicyEngine leaves $194.38 of monthly net income, so the maximum-allotment shortcut does not apply." +us,scenario_118,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model denied a positive benefit despite the household’s very low income and assets and incorrectly described the case as having no relevant expenses. PolicyEngine finds categorical eligibility through SSI and TANF non-cash eligibility and calculates a positive monthly allotment. +us,scenario_118,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of additional supporting facts as a bar to SNAP. The supplied facts and PolicyEngine-generated SSI and TANF pathways establish eligibility, while unlisted facts default to zero rather than preventing calculation." +us,scenario_118,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model invented a $20 SSI/SNAP exclusion, reduced net income to zero, and used $243 as the maximum allotment. PolicyEngine uses a $298 maximum for most months and subtracts 30% of $194.38 net income, with updated parameters in other months." +us,scenario_118,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,The answer reflects a near-maximum allotment based only on Social Security income and a standard deduction. It omits modeled SSI from countable income and the resulting $194.38 net-income amount that triggers the $58.20 expected contribution. +us,scenario_118,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model estimated a $115 monthly benefit from unspecified Social Security, SSI, standard, and shelter figures instead of applying the traced calculation. PolicyEngine derives $194.38 of net income and generally pays $239.80 per month, not $115." +us,scenario_118,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,The model’s $123 monthly estimate results from an incorrect treatment of modeled SSI and shelter deductions. PolicyEngine’s applicable deductions leave $194.38 of net income and generally yield $239.80 per month. +us,scenario_118,snap,grok-4.3,llm_error,categorical_eligibility,False,The model treated the listed facts as inadequate to calculate SNAP and defaulted to zero. PolicyEngine determines categorical eligibility through SSI receipt and TANF non-cash eligibility and computes a positive allotment from the supplied household data. +us,scenario_118,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model recognized SSI-based categorical eligibility but produced about $735 of net income after claiming unlimited shelter deductions. PolicyEngine’s income and deduction sequence produces $194.38 of net income, so its expected contribution and benefit are substantially different." +us,scenario_118,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly set shelter costs to zero even though mortgage interest and real estate taxes were listed, and it counted only Social Security income. PolicyEngine also includes modeled SSI, applies its SNAP deductions, and arrives at $194.38 of net income rather than $40." +us,scenario_118,snap,inkling,llm_error,taxable_income_or_deductions,False,The model used mortgage interest and property taxes to reduce net income to zero and awarded twelve months of the maximum allotment. PolicyEngine leaves $194.38 of net income and subtracts the corresponding 30% expected contribution. +us,scenario_118,snap,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model treated the full ownership costs as an uncapped shelter deduction that eliminated net income, then used an incorrect $309 maximum allotment. PolicyEngine’s deductions leave $194.38 of net income and its maximum is $298 for most months." +us,scenario_118,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used $10,412 of annual SSI instead of PolicyEngine’s $780.67 monthly SSI amount and ignored the listed mortgage interest and property taxes because no separate shelter field was present. Those errors inflated net income to $892 instead of the traced $194.38." +us,scenario_118,snap,minimax-m3,llm_error,thresholds_rates,False,The model reversed the SNAP benefit formula by concluding that income below the threshold produces zero after deductions. Low net income increases the allotment; PolicyEngine calculates $194.38 of net income and a positive benefit after the 30% contribution. +us,scenario_118,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model counted only Social Security, omitted modeled SSI, and applied only the standard deduction, leaving about $35 of net income. PolicyEngine includes $780.67 of SSI receipt and the full applicable deduction sequence, producing $194.38 of net income and month-specific allotments." +us,scenario_118,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly required a qualifying child or another household member for SNAP. A one-person elderly or disabled household qualifies independently, and this household passes the income and asset tests and is categorically eligible through SSI and TANF non-cash eligibility." +us,scenario_118,ssi,claude-fable-5,llm_error,other,False,"The model explicitly derived $994 - $213.33 = $780.67 monthly and $9,368 annually, then discarded that calculation and submitted $6,144. Its final value does not follow from its own stated SSI rate or income reduction." +us,scenario_118,ssi,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model compared $2,800 of annual Social Security income with a monthly SSI limit and incorrectly declared the individual income-ineligible. Monthly countable income is only $213.33 after the $20 exclusion, below the $994 monthly maximum, so SSI remains payable." +us,scenario_118,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a $967 federal monthly rate instead of $994 and added an unsupported New York supplement to the requested `ssi` output. The correct calculation contains no separate state supplement and yields 12 × ($994 - $213.33) = $9,368." +us,scenario_118,ssi,claude-opus-4.8,llm_error,other,False,"The model correctly concluded that resources pass and that federal SSI is positive after deducting countable Social Security income, but then submitted zero. It failed to carry its positive-benefit calculation into the output." +us,scenario_118,ssi,claude-opus-5,llm_error,thresholds_rates,False,"The model stated the correct $213.33 monthly countable income but reduced the resulting payment to roughly $667 without a valid rule or arithmetic step. Applying that income to the $994 maximum gives $780.67 monthly, not $667." +us,scenario_118,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the obsolete $967 monthly federal rate instead of the 2026 $994 rate. It also initially treated the $20 general exclusion as annual before switching calculations; the correct $20 monthly exclusion leaves $213.33 monthly countable income and produces $9,368." +us,scenario_118,ssi,claude-sonnet-5,llm_error,other,False,"The model's stated figures produce $12,152 - $2,780 = $9,372, yet it submitted $6,212 after an unexplained adjustment. No SSI income-counting or New York rule supports that reduction." +us,scenario_118,ssi,deepseek-v4-pro,llm_error,thresholds_rates,False,The model used $991 rather than the $994 monthly maximum and added a $54 New York supplement that is not part of the requested PolicyEngine `ssi` output. SSI is calculated solely as 12 × ($994 - $213.33). +us,scenario_118,ssi,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $991 federal rate and added an $87 New York living-alone supplement. The requested `ssi` value uses the $994 maximum without that separate supplement, producing $9,368." +us,scenario_118,ssi,gemini-3.1-flash-lite-preview,llm_error,period_annualization,False,"The model asserted that $2,800 in annual Social Security exceeds the SSI maximum, conflating an annual income figure with the monthly benefit threshold. Social Security is $233.33 monthly and $213.33 after the monthly exclusion, well below the $994 maximum." +us,scenario_118,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model added a $1,044 New York supplement and used an $11,316 federal maximum instead of the traced $11,928 annual maximum. Excluding the separate state supplement and subtracting $2,560 of countable income gives $9,368." +us,scenario_118,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model substituted the 2024 $943 monthly rate and added an $87 monthly New York supplement. For 2026 the applicable maximum is $994 monthly, and the requested output does not add that supplement." +us,scenario_118,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly calculated $2,560 of annual countable income but used an annual maximum of $11,796. The 2026 maximum is $11,928, so subtracting $2,560 yields $9,368." +us,scenario_118,ssi,glm-5.2,llm_error,thresholds_rates,False,"The model estimated a $991 federal rate and added a $94 monthly New York supplement. The applicable maximum is $994 per month, with no separate state supplement in this output, so the annual result is $9,368." +us,scenario_118,ssi,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model ignored the supplied age, blindness, disability, $70 resource amount, and $2,800 Social Security income, all of which establish the SSI calculation. The individual qualifies as aged, blind, and disabled, passes the resource test, and has only $213.33 in monthly countable income." +us,scenario_118,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed the necessary eligibility facts were absent even though age 74, blindness, disability, $70 in bank assets, and Social Security income were explicitly listed. Those facts establish categorical and resource eligibility and leave a positive income-adjusted payment." +us,scenario_118,ssi,gpt-5.5,llm_error,thresholds_rates,False,"The model handled the $240 annual exclusion correctly but used a $982.67 monthly maximum instead of $994. The correct annual maximum is $11,928, and $11,928 - $2,560 = $9,368." +us,scenario_118,ssi,gpt-5.6-terra,llm_error,period_annualization,False,"The model treated the $20 general income exclusion as a one-time annual exclusion, subtracting only $20 from $2,800. The exclusion applies monthly, totaling $240 annually, so countable income is $2,560 and SSI is $9,368." +us,scenario_118,ssi,grok-4.3,llm_error,categorical_eligibility,False,"The model asserted that SSI calculation inputs were absent, overlooking the stated age, blindness, disability, bank assets, and Social Security income. These inputs establish eligibility and yield $9,368 after the monthly income exclusion and reduction." +us,scenario_118,ssi,grok-4.5,llm_error,thresholds_rates,False,"The model used a $991 federal maximum and added an $87 New York supplement. The requested calculation uses $994 monthly without the separate supplement, leaving $780.67 monthly after countable income." +us,scenario_118,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"The model used the 2024 $943 federal maximum as a proxy and added an $87 New York supplement. The benchmark year requires the 2026 $994 maximum, and the requested `ssi` output excludes that separate supplement." +us,scenario_118,ssi,inkling,llm_error,thresholds_rates,False,"The model correctly derived about $780.67 monthly from the $994 maximum and $213.33 countable income, then incorrectly added an $87 monthly New York supplement. The requested PolicyEngine `ssi` amount is the $9,368 federal calculation without that addition." +us,scenario_118,ssi,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly calculated $2,560 in annual countable income but used a $991 monthly maximum. Using $994 gives an annual maximum of $11,928 and a benefit of $9,368." +us,scenario_118,ssi,kimi-k3,llm_error,thresholds_rates,False,"The model correctly derived the $9,368 federal SSI amount, then added a $1,044 New York living-alone supplement. That supplement is not included in the requested PolicyEngine `ssi` output." +us,scenario_118,ssi,minimax-m3,llm_error,thresholds_rates,False,"The model stated that $233 monthly Social Security exceeds a $967 monthly SSI rate, reversing the numerical comparison. After the $20 exclusion, countable income is $213.33 monthly, which reduces rather than eliminates the $994 maximum." +us,scenario_118,ssi,qwen-3.7-max,llm_error,other,False,"The model calculated a positive monthly and annual SSI benefit and then submitted zero. Its eligibility findings and income calculation directly require a positive output; using the 2026 $994 rate raises that amount to $9,368." +us,scenario_118,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"The model denied a positive payment despite the explicit aged, blind, and disabled status, resources below $2,000, and low countable income. These facts establish eligibility, and subtracting $213.33 monthly countable income from $994 leaves $780.67 payable each month." +us,scenario_118,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model wrongly excluded the homeowner from New York's real property tax credit after correctly identifying the absence of EITC and child-related credits. Age 74 and $1,633.50 of real estate taxes satisfy the relevant senior-homeowner pathway, producing a $375 refundable credit." +us,scenario_118,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model overlooked New York's refundable real property tax credit and incorrectly treated refundable credits as limited to programs such as the state EITC. The senior homeowner's age and property taxes generate $375 despite Social Security being exempt from state income tax. +us,scenario_118,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,The model identified the real property tax credit and its $375 senior maximum but then applied an unsupported approximate formula that reduced it to $63. The applicable calculation reaches the full $375 maximum. +us,scenario_118,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly made positive New York tax liability a prerequisite for a refundable real property tax credit. The credit is refundable, and the qualifying 74-year-old homeowner receives $375 even with no taxable New York income." +us,scenario_118,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated the absence of rent and taxable New York income as disqualifying, ignoring the separate homeowner pathway based on property taxes paid. The age-74 homeowner's $1,633.50 property-tax burden yields the $375 refundable real property tax credit." +us,scenario_118,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly found eligibility and calculated a preliminary credit above the cap, but it applied the $75 maximum for a non-senior single filer. Because the taxpayer is 74, the applicable maximum is $375, not $75." +us,scenario_118,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model dismissed the real property tax credit because there was no rent and focused on earned-income and child credits. Homeowners qualify through real estate taxes rather than rent, and this senior homeowner reaches the $375 credit maximum." +us,scenario_118,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model correctly included the $375 real property tax credit but incorrectly added a $75 New York household credit to refundable credits. The household credit is nonrefundable and therefore does not belong in state_refundable_credits. +us,scenario_118,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model denied all New York refundable-credit eligibility without applying the real property tax credit's senior-homeowner pathway. Age 74 and $1,633.50 of real estate taxes produce a $375 credit." +us,scenario_118,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model considered only the state EITC and child-related credits, then inferred that earned income or dependents were required for every refundable credit. New York's real property tax credit instead uses the taxpayer's senior status and homeowner property-tax burden, yielding $375." +us,scenario_118,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model failed to evaluate New York's refundable real property tax credit. The 74-year-old New York homeowner with $1,633.50 of real estate taxes qualifies for $375." +us,scenario_118,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an inapplicable 3.5%-of-income excess-tax formula and an unsupported $1,000 expanded senior maximum. The applicable New York real property tax credit calculation is capped at $375 for this senior household." +us,scenario_118,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model's zero answer omits the refundable New York real property tax credit triggered by the listed age and real estate taxes. The senior-homeowner calculation yields $375. +us,scenario_118,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no listed fact indicated a refundable credit. Age 74, New York residency, and $1,633.50 of property taxes are the facts that establish the $375 real property tax credit." +us,scenario_118,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model treated Social Security-only income as preventing all refundable New York credits. The real property tax credit does not require earnings or positive income-tax liability, and the senior homeowner receives $375." +us,scenario_118,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model substituted a $90 New York household credit for the applicable refundable real property tax credit. The household credit is not part of refundable credits, while the age-74 homeowner's property-tax facts generate $375." +us,scenario_118,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model's blanket denial of qualifying credits omits the senior-homeowner real property tax credit. Applying the age and property-tax conditions produces $375. +us,scenario_118,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model treated the state EITC and child credits as the only relevant refundable programs. It omitted New York's real property tax credit, under which the 74-year-old homeowner receives $375." +us,scenario_118,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,The model asserted that no property tax credit applied without using the listed age and real estate taxes. Those facts qualify the homeowner for New York's $375 refundable real property tax credit. +us,scenario_118,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model exhaustively checked earned-income, child, tuition, and childcare credits but omitted the real property tax credit. That credit requires neither children nor earnings, and this senior homeowner's property-tax burden produces $375." +us,scenario_118,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,The model incorrectly treated zero New York tax liability and the absence of children as disqualifying for every refundable credit. The real property tax credit is refundable and awards this qualifying senior homeowner $375. +us,scenario_118,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model focused on EITC and child-credit conditions and ignored the listed eligibility factors for New York's real property tax credit. The taxpayer's age and $1,633.50 property-tax payment produce a $375 refundable credit." +us,scenario_118,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model incorrectly concluded that age and household facts supported no refundable credit. Age 74 and the homeowner's real estate taxes establish eligibility for the $375 New York real property tax credit without earned income or dependents. +us,scenario_119,child1_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a 205% FPL ceiling instead of the applicable 200% FPL CHIP ceiling. At approximately $55,800 of household income, Child 1 passes the under-19 age test but fails Virginia CHIP's income test." +us,scenario_119,child1_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model asserted that household income was within Virginia's CHIP threshold without comparing the engine-derived income of approximately $55,800 against the applicable 200% FPL ceiling. That comparison fails, despite Child 1 satisfying the age condition." +us,scenario_119,child1_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly placed the household above the Medicaid child limit but inside Virginia's CHIP income range. The approximately $55,800 household income exceeds the applicable CHIP ceiling, so Medicaid ineligibility does not lead to CHIP eligibility." +us,scenario_119,child1_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model correctly applied the under-19 age condition but incorrectly classified the household income as within Virginia's CHIP range. The engine-derived income of approximately $55,800 is above the applicable 200% FPL ceiling." +us,scenario_119,child1_chip_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model understated the household's FPL ratio as approximately 192%. Using the engine-derived household income of approximately $55,800 puts the household above Virginia CHIP's applicable 200% FPL ceiling." +us,scenario_119,child1_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used CHIP MAGI of $46,180 instead of the engine-derived household income of approximately $55,800. That understatement placed Child 1 inside the claimed 143–200% FPL band, while the correct income exceeds its upper bound." +us,scenario_119,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for child1_chip_eligible. The required output is 0 because the approximately $55,800 household income exceeds Virginia CHIP's applicable income ceiling." +us,scenario_119,child1_chip_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used household MAGI of $52,569 instead of the engine-derived household income of approximately $55,800. This understated income produced a false placement within Virginia's CHIP band; the correct income exceeds the applicable ceiling." +us,scenario_119,child1_medicaid_eligible,claude-fable-5,llm_error,health_coverage,False,"The model correctly found income above Virginia's child Medicaid limit, then incorrectly treated the higher FAMIS/CHIP threshold as a Medicaid-expansion pathway. FAMIS eligibility is reported through CHIP, while the Medicaid category remains NONE at 1.92 times FPL." +us,scenario_119,child1_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used a family of four even though the listed household contains one head and two children, and it merged Virginia Medicaid and FAMIS/CHIP into one 205% FPL threshold. The engine's three-person household calculation produces 1.92 times FPL, above the applicable Medicaid limit, with no Medicaid category." +us,scenario_119,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that 1.92-times-FPL MAGI falls within Virginia's child Medicaid limit. That income exceeds the applicable Medicaid threshold for a 14-year-old, and no alternative Medicaid pathway applies." us,scenario_119,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for child1_medicaid_eligible, violating the required output contract." us,scenario_119,child1_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_119,child2_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model treated 205% FPL as a qualifying Virginia FAMIS ceiling and concluded that approximately $55,800 fell below it. Under the applicable PolicyEngine income test, that household income exceeds the CHIP limit; age 11 satisfies the age condition but does not overcome the failed income condition." -us,scenario_119,child2_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model asserted that the household income was within Virginia's CHIP threshold without applying the applicable limit to PolicyEngine's approximately $55,800 income figure. Child 2 passes the age condition but fails CHIP's income condition." -us,scenario_119,child2_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model placed approximately $55,800 within the 2026 Virginia CHIP/FAMIS child income limit. PolicyEngine's applicable income threshold excludes the child at that income, despite the child satisfying the under-19 age rule." -us,scenario_119,child2_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model relied on an estimated Virginia CHIP income range instead of the applicable PolicyEngine threshold. The child is age-qualified, but approximately $55,800 exceeds the program's income limit." -us,scenario_119,child2_chip_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model understated the household's income-to-FPL ratio as approximately 192% and therefore placed it below a 200% FPL ceiling. PolicyEngine applies approximately $55,800 of household income, which fails Virginia CHIP's income test." -us,scenario_119,child2_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the CHIP income measure to $46,180 and 168% FPL, placing the child inside a claimed 143–200% band. PolicyEngine's CHIP determination uses approximately $55,800 of household income, which exceeds the applicable limit." -us,scenario_119,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for child2_chip_eligible. The required output was 0 because the age-11 child fails Virginia CHIP's income condition at approximately $55,800 of household income." -us,scenario_119,child2_chip_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model substituted a reduced MAGI of $52,569 and treated it as falling within Virginia's CHIP income band. PolicyEngine applies approximately $55,800 in the CHIP income test, which puts the child above the applicable limit even though the child is under 19." -us,scenario_119,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly calculated income near $55,800 and even recognized that it exceeded its asserted Medicaid threshold, but then reversed its conclusion by combining Medicaid and CHIP rules. PolicyEngine assigns child2 no Medicaid category at 1.92 times FPL; a separate FAMIS/CHIP pathway does not make child2 Medicaid-eligible." -us,scenario_119,child2_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated Virginia Medicaid and FAMIS/CHIP as one eligibility pathway and applied an approximate 205%-FPL combined coverage ceiling to the Medicaid output. Child2 qualifies through no Medicaid category at 1.92 times FPL, and asset disregard does not change that categorical result." -us,scenario_119,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model asserted that 1.92-times-FPL MAGI was within Virginia's child Medicaid limit without applying the applicable Medicaid threshold for an 11-year-old. The threshold comparison yields no Medicaid category and therefore a No result. -us,scenario_119,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child2_medicaid_eligible, violating the required structured-output contract." +us,scenario_119,child2_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model treated the household as below Virginia's 205% FPL limit without comparing the applicable threshold against PolicyEngine's household income of approximately $55,800. Child 2 meets the age condition but fails CHIP's income test." +us,scenario_119,child2_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model asserted that household income falls within Virginia's CHIP threshold, but approximately $55,800 exceeds the applicable limit. Age 11 satisfies CHIP's age condition, leaving the income test as the disqualifying step." +us,scenario_119,child2_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model placed the household within the 2026 Virginia FAMIS child income limit when PolicyEngine's approximately $55,800 household income is above it. It correctly recognized the child's age category but misapplied the income threshold." +us,scenario_119,child2_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model relied on an estimated Virginia CHIP income range and incorrectly put approximately $55,800 inside that range. Child 2 is under 19, but the failed income test makes the child ineligible." +us,scenario_119,child2_chip_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model understated the household's income-to-FPL ratio as approximately 192% and therefore placed it below a 200% FPL ceiling. PolicyEngine uses approximately $55,800, which exceeds Virginia's applicable CHIP income limit for this household." +us,scenario_119,child2_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model reduced CHIP MAGI to $46,180, improperly excluding amounts from the household income used by PolicyEngine's eligibility test. With approximately $55,800 counted, Child 2 is above Virginia's CHIP income limit despite satisfying the age rule." +us,scenario_119,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no child2_chip_eligible value or explanation. The required output is 0 because the 11-year-old satisfies the age condition but the household's approximately $55,800 income exceeds Virginia's CHIP limit." +us,scenario_119,child2_chip_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used MAGI of $52,569 instead of PolicyEngine's approximately $55,800 household income and consequently placed the household inside Virginia's CHIP band. At the income used by the eligibility computation, Child 2 exceeds the applicable limit." +us,scenario_119,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly calculated that income exceeded the Medicaid limit, then reversed its conclusion by combining Medicaid with CHIP/FAMIS and treating possible CHIP eligibility as Medicaid eligibility. Child2 qualifies through no Medicaid pathway, so proximity to a broader children's coverage threshold cannot produce Medicaid eligibility." +us,scenario_119,child2_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated Virginia children's Medicaid with FAMIS/CHIP and applied a combined threshold of about 205% FPL to the Medicaid output. At 1.92 times FPL, child2 falls into no Medicaid category; eligibility under a separate FAMIS/CHIP pathway does not make is_medicaid_eligible true." +us,scenario_119,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that 1.92-times-FPL MAGI was within Virginia's Medicaid income limit for an 11-year-old. That income exceeds every applicable Medicaid threshold for child2, leaving medicaid_category NONE and Medicaid eligibility false." +us,scenario_119,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child2_medicaid_eligible, violating the required output contract." us,scenario_119,child2_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_119,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model incorrectly restricted the nonrefundable CTC to about $800 by treating the refundable CTC cap as forcing a residual tax liability. The correct allocation uses $2,310.23 of nonrefundable CTC after the $546 CDCC and $200 Saver's Credit, reducing $3,056.23 of tax to zero." -us,scenario_119,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model used single filing status and a $14,600 deduction instead of head-of-household status and the $24,150 deduction. It also wrongly denied the CDCC because employer-sponsored health insurance exists, even though health coverage has no bearing on the dependent-care credit, and omitted the Saver's Credit and the correct refundable/nonrefundable CTC allocation." -us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model incorrectly treated the 14-year-old as eligible only for the $500 other-dependent credit, even though both children are under 17 and qualify for the CTC. It also used a $312 CDCC instead of $546 and omitted the $200 Saver's Credit, leaving a residual that the correct nonrefundable CTC eliminates." -us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model explicitly recognized that the available CTC could eliminate the remaining liability, then submitted $1,568 by applying only an unexplained partial amount. The actual nonrefundable credits are $546 of CDCC, $200 of Saver's Credit, and $2,310.23 of CTC, which exhaust the $3,056.23 liability." -us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model applied only about $800 of nonrefundable CTC and omitted both the $546 CDCC and $200 Saver's Credit. The refundable CTC amount does not cap the nonrefundable CTC at $800; $2,310.23 is used nonrefundably to eliminate the liability remaining after the other credits." -us,scenario_119,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model abandoned its initial zero result and applied an obsolete assumed post-TCJA-expiration regime with personal exemptions, pre-TCJA brackets, and a $1,000-per-child CTC. For 2026 current law, the household receives the $24,150 head-of-household standard deduction and the applicable credit stack of $546 CDCC, $200 Saver's Credit, and $2,310.23 nonrefundable CTC, reducing tax to zero." -us,scenario_119,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model computed a positive residual while simultaneously stating that the $4,000 potential CTC exceeded pre-credit tax, an internally inconsistent credit application. It also omitted the $546 CDCC and $200 Saver's Credit; after those credits, $2,310.23 of nonrefundable CTC eliminates the remaining liability." -us,scenario_119,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used only $1,000 per child for the CTC, applied a $312 CDCC, and omitted the $200 Saver's Credit. The correct nonrefundable amounts are $2,310.23 of CTC, $546 of CDCC, and $200 of Saver's Credit, exactly offsetting $3,056.23 of tax." -us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model miscomputed AGI as $51,713 instead of $52,568.54 and used an understated approximate head-of-household deduction rather than $24,150. More importantly, its explanation reports bracket tax as the final output without subtracting the $546 CDCC, $200 Saver's Credit, and $2,310.23 nonrefundable CTC." -us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model invented an AGI of $46,180 and personal exemptions, neither of which follows from the stated inputs or 2026 current law. It also applied only $2,000 of CTC and $312 of CDCC while omitting the $200 Saver's Credit; the correct nonrefundable-credit stack fully offsets tax." -us,scenario_119,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The submitted $6,350 exceeds even the traced $3,056.23 income tax before credits, so the model did not correctly calculate the tax base or subtract the available nonrefundable credits. The $546 CDCC, $200 Saver's Credit, and $2,310.23 nonrefundable CTC reduce the traced liability to zero." -us,scenario_119,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly used an assumed post-TCJA-expiration system of personal exemptions and understated regular tax at about $2,755. It then used only a $312 CDCC and $2,000 CTC, omitting the $200 Saver's Credit and the correct $2,310.23 nonrefundable CTC allocation that eliminates the traced liability." -us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model's $2,492 answer is consistent with subtracting only a small portion of the available nonrefundable credits and provides no credit-by-credit reconciliation. The traced liability of $3,056.23 is fully offset by $546 of CDCC, $200 of Saver's Credit, and $2,310.23 of nonrefundable CTC." -us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-expiration regime with personal exemptions, 10%/15% brackets, and a $2,000 total CTC. Under 2026 current law, tax before credits is $3,056.23 and the applicable $546 CDCC, $200 Saver's Credit, and $2,310.23 nonrefundable CTC reduce it to zero." -us,scenario_119,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response failed the submission contract." -us,scenario_119,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model incorrectly set the EITC to zero instead of applying the two-child phaseout, which leaves $764.23. It also treated the refundable CTC as the full $3,400 per-child-cap total rather than the $2,089.77 remaining after the nonrefundable-credit calculation." -us,scenario_119,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly treated this income as exceeding both refundable-credit eligibility ranges. The two-child EITC phaseout still yields $764.23, and the refundable CTC is $2,089.77." -us,scenario_119,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model prematurely reduced the EITC to zero near the phaseout endpoint and understated the CTC remaining after nonrefundable credits. The correct components are $764.23 of EITC and $2,089.77 of refundable CTC, not approximately $900 of ACTC alone." -us,scenario_119,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $48,300 despite also claiming a $32,300 deduction from roughly $52,569 of AGI, an arithmetic inconsistency that inflated pre-credit tax. It therefore incorrectly concluded that tax absorbed the entire CTC and also eliminated the $764.23 residual EITC." -us,scenario_119,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly set EITC to zero and assumed a $1,800-per-child refundable CTC maximum was fully available. Applying the actual phaseout and nonrefundable-credit calculation produces $764.23 of EITC and $2,089.77 of refundable CTC." -us,scenario_119,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model used estimated rather than actual 2026 EITC parameters and tax calculations, producing about $922 of EITC and only $723 of refundable CTC. The applicable calculations yield $764.23 and $2,089.77, respectively." -us,scenario_119,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model incorrectly concluded that regular tax fully absorbed the CTC and that income exceeded the two-child EITC range. The nonrefundable-credit calculation leaves $2,089.77 refundable, while the EITC phaseout leaves $764.23." -us,scenario_119,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used the wrong EITC maximum and phaseout parameters, yielding $1,431 instead of $764.23. It also incorrectly declared the CTC fully used nonrefundably, omitting $2,089.77 of refundable CTC." -us,scenario_119,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $46,180 without support from the listed deductible contributions, substantially overstating EITC. It also understated the refundable CTC at $427 instead of $2,089.77." -us,scenario_119,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model assigned a round $3,000 refundable CTC without calculating its interaction with tax liability and nonrefundable credits, and it omitted EITC entirely. The components are $2,089.77 of refundable CTC and $764.23 of EITC." -us,scenario_119,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model overstated EITC at $1,933 and understated the refundable CTC at $172. Applying the 2026 two-child phaseout and the nonrefundable-credit ordering yields $764.23 and $2,089.77." -us,scenario_119,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used unsupported earned income of $45,524, inflating EITC to $1,808 instead of $764.23. It also incorrectly treated the CTC as fully nonrefundable and omitted the $2,089.77 refundable portion." -us,scenario_119,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The submitted total does not equal the calculated components of $764.23 EITC and $2,089.77 refundable CTC. Its generic explanation omits the phaseout and nonrefundable-credit steps necessary to reach $2,854.01." -us,scenario_119,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model incorrectly treated earned income near $51,913 as beyond the two-child EITC endpoint, omitting $764.23. It also overstated the CTC used nonrefundably and therefore reduced the refundable CTC to $1,106 instead of $2,089.77." -us,scenario_119,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated the entire $2,000-per-child CTC as refundable without first applying the nonrefundable portion against tax liability. That calculation leaves $2,089.77 refundable, and the model also omitted the $764.23 EITC." -us,scenario_119,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The unexplained $190 estimate does not apply either required component calculation. The two-child EITC is $764.23 and the refundable CTC is $2,089.77." -us,scenario_119,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model used incorrect EITC phaseout parameters, producing $1,496 instead of $764.23. It also incorrectly treated the CTC as fully used nonrefundably, omitting $2,089.77." -us,scenario_119,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model substantially overstated EITC at $3,000 rather than applying the phaseout to reach $764.23. It also overstated the refundable CTC at $2,363.10 instead of $2,089.77." -us,scenario_119,federal_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model overstated EITC at $1,276 and understated refundable CTC at $1,656. The applicable calculations produce $764.23 and $2,089.77." -us,scenario_119,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model approximated EITC closely but still used $785 instead of $764.23, and it understated refundable CTC at $1,656 rather than $2,089.77. Those component errors reduced the total to $2,441." -us,scenario_119,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly treated income as above the two-child EITC phaseout endpoint and ignored the refundable CTC. The household receives $764.23 of EITC and $2,089.77 of refundable CTC." -us,scenario_119,federal_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model used incorrect EITC parameters, yielding $1,300 instead of $764.23. It also treated the CTC as fully absorbed nonrefundably and omitted the $2,089.77 refundable portion." -us,scenario_119,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model failed to apply the two-child EITC phaseout correctly, overstating EITC at $4,109 instead of $764.23. It also understated refundable CTC at $584 instead of $2,089.77." -us,scenario_119,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for federal_refundable_credits. -us,scenario_119,federal_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model correctly calculated the $764.23 EITC but overstated the CTC used nonrefundably at $3,816.28. The nonrefundable-credit calculation leaves $2,089.77 of refundable CTC, not $583.72." -us,scenario_119,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly treated $55,000 of wages as exceeding the two-child EITC range and treated the CTC as fully absorbed by tax. The correct residual amounts are $764.23 of EITC and $2,089.77 of refundable CTC." -us,scenario_119,federal_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly used married-filing-jointly EITC thresholds for an unmarried head-of-household filer and treated the phaseout-start threshold as though the maximum credit remained intact until that point. This inflated EITC to about $6,400 instead of $764.23; the refundable CTC is $2,089.77, not $1,837." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model invented an $800 limit on the nonrefundable CTC and treated the refundable CTC allocation as leaving $1,210 of tax. After the $546 CDCC and $200 Saver's Credit, $2,310.23 of nonrefundable CTC offsets all remaining liability." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model used single filing status and its standard deduction instead of head-of-household status, then incorrectly denied the CDCC because the head has employer-sponsored health insurance. It also omitted the Saver's Credit and failed to allocate the CTC between its nonrefundable and refundable portions." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model incorrectly treated the 14-year-old as eligible only for the $500 Credit for Other Dependents rather than the Child Tax Credit. It also understated the CDCC at $312 instead of $546 and omitted the $200 Saver's Credit, leaving a false residual liability." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model explicitly found that available nonrefundable CTC could eliminate the remaining tax, then submitted $1,568 without a supporting computation. It also understated CDCC at $312 and omitted the $200 Saver's Credit." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model arbitrarily limited the nonrefundable CTC to about $800 and omitted both the $546 CDCC and $200 Saver's Credit. The nonrefundable CTC actually used is $2,310.23, exactly exhausting the liability remaining after those credits." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied a hypothetical expiration of the post-TCJA tax structure, including personal exemptions, old brackets, and a $1,000 CTC, instead of the applicable 2026 parameters. It also constructed unsupported medical itemized deductions from premiums and expenses and omitted the Saver's Credit." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model did not actually subtract the available nonrefundable credits from its computed tax, despite acknowledging that the CTC alone was sufficient to cover it. It also omitted the $546 CDCC and $200 Saver's Credit and supplied a residual unsupported by its arithmetic." +us,scenario_119,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used a reverted $1,000-per-child CTC, limiting the total CTC to $2,000. It also understated CDCC at $312 and omitted the $200 Saver's Credit; the applicable nonrefundable CTC is $2,310.23 after the other credits." +us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model miscomputed AGI as $51,713 instead of $52,568.54 and used an understated head-of-household standard deduction. It then failed to subtract the $546 CDCC, $200 Saver's Credit, and $2,310.23 nonrefundable CTC that eliminate the tax." +us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $46,180 without support from the listed inputs and applied personal exemptions that do not belong in the reference computation. It also used only $2,000 of CTC, understated CDCC at $312, and omitted the $200 Saver's Credit." +us,scenario_119,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The submitted $6,350 exceeds even the traced $3,056.23 income tax before credits and is incompatible with the model's claim that it subtracted nonrefundable credits. It failed to apply the $546 CDCC, $200 Saver's Credit, and $2,310.23 nonrefundable CTC that reduce the output to zero." +us,scenario_119,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an obsolete post-TCJA-expiration structure with personal exemptions and understated the CTC at $2,000. It also understated CDCC at $312 and omitted the $200 Saver's Credit, instead of applying the credits that fully offset $3,056.23 of tax." +us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated the mortgage balance as supporting a mortgage-interest deduction even though no mortgage interest was listed and unlisted numeric inputs were zero. It also failed to apply the full traced combination of CDCC, Saver's Credit, and nonrefundable CTC that eliminates the liability." +us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly reverted to personal exemptions and old 10%/15% brackets for 2026 and limited the CTC to $2,000. Although it identified the Saver's Credit, it understated CDCC at $312 and missed the $2,310.23 nonrefundable CTC actually used." +us,scenario_119,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so its response failed the required output contract." +us,scenario_119,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model incorrectly phased the EITC down to zero instead of $764.23 and treated the refundable CTC as the full $1,700-per-child cap. The refundable CTC is limited by the child credit remaining after nonrefundable use and equals $2,089.77, not $3,400." +us,scenario_119,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model wrongly treated this income as eliminating both credits. The two-child EITC phaseout still leaves $764.23, and the CTC refundability rules leave $2,089.77 refundable after the nonrefundable portion is used." +us,scenario_119,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model incorrectly reduced the EITC to zero and overstated the regular tax absorbed by the CTC. The correct credit allocation produces a $764.23 EITC and $2,089.77 refundable CTC rather than approximately $900 of refundable CTC alone." +us,scenario_119,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $48,300 despite describing deductions that place taxable income far lower, causing it to assert that tax fully absorbs the CTC. It also eliminated the EITC; the correct components are $764.23 of EITC and $2,089.77 of refundable CTC." +us,scenario_119,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model wrongly set EITC to zero and treated the per-child refundable cap as the refundable CTC calculation itself. After allocating the CTC against tax, only $2,089.77 is refundable, and the EITC contributes another $764.23." +us,scenario_119,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model estimated 2026 parameters and rounded instead of applying the exact EITC phaseout and CTC allocation. Those computations yield a $764.23 EITC and $2,089.77 refundable CTC, not roughly $922 and $723." +us,scenario_119,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model incorrectly concluded that regular tax fully absorbs the CTC and that income exceeds the two-child EITC range. The household retains $2,089.77 of refundable CTC and $764.23 of EITC." +us,scenario_119,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used incorrect 2026 EITC parameters, producing $1,431 instead of $764.23, and incorrectly assumed the entire CTC was used nonrefundably. The unused refundable CTC is $2,089.77." +us,scenario_119,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used an unsupported AGI of $46,180, which inflated EITC to $2,643, and then left only $427 of refundable CTC. Applying the listed income and deductions produces the traced $764.23 EITC and $2,089.77 refundable CTC." +us,scenario_119,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model substituted a flat $3,000 refundable CTC estimate and omitted the EITC. The CTC split leaves $2,089.77 refundable, and the two-child EITC adds $764.23." +us,scenario_119,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model overstated the EITC at $1,933 and understated the refundable CTC at $172. The exact phaseout and nonrefundable-credit ordering yield $764.23 and $2,089.77, respectively." +us,scenario_119,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used unsupported earned income of $45,524, inflating the EITC to $1,808, and incorrectly concluded that no CTC remained refundable. The correct components are $764.23 of EITC and $2,089.77 of refundable CTC." +us,scenario_119,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The submitted total does not implement the traced component calculations. Applying the exact two-child EITC phaseout and refundable CTC split gives $764.23 plus $2,089.77, totaling $2,854.01." +us,scenario_119,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model incorrectly treated earned income near $51,913 as beyond the EITC phaseout endpoint and overstated the CTC used nonrefundably. EITC remains $764.23, while the refundable CTC is $2,089.77 rather than $1,106." +us,scenario_119,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated the entire $2,000-per-child CTC as refundable without first allocating part of it nonrefundably against tax, and it omitted EITC. The refundable CTC is $2,089.77 and EITC is $764.23." +us,scenario_119,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The unexplained $190 estimate omits nearly all of both refundable-credit components. The exact derivation yields $764.23 of EITC and $2,089.77 of refundable CTC." +us,scenario_119,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model incorrectly concluded that the CTC was fully used nonrefundably and used the wrong EITC phaseout result. The household receives $2,089.77 of refundable CTC and $764.23 of EITC." +us,scenario_119,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model overstated both refundable components, assigning about $3,000 to EITC and $2,363.10 to refundable CTC. The exact amounts are $764.23 and $2,089.77." +us,scenario_119,federal_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model understated refundable CTC at $1,656 and overstated EITC at $1,276. The correct CTC allocation and EITC phaseout produce $2,089.77 and $764.23." +us,scenario_119,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The EITC estimate is close, but the model left only $1,656 of refundable CTC by overstating the child credit used nonrefundably. The correct amounts are $764.23 of EITC and $2,089.77 of refundable CTC." +us,scenario_119,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly placed the household above the two-child EITC phaseout endpoint and omitted the refundable CTC pathway. The household receives $764.23 of EITC and $2,089.77 of refundable CTC." +us,scenario_119,federal_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model used incorrect EITC parameters to obtain $1,300 and incorrectly treated the entire CTC as absorbed nonrefundably. The exact calculations yield $764.23 of EITC and $2,089.77 of refundable CTC." +us,scenario_119,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model greatly overstated EITC at $4,109 and understated refundable CTC at $584. Applying the two-child EITC phaseout and correct CTC allocation yields $764.23 and $2,089.77." +us,scenario_119,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The near-total match is coincidental because both components are wrong: it used a $4,400 total CTC and calculated EITC as $1,188. The traced 2026 rules produce $2,089.77 of refundable CTC and $764.23 of EITC." +us,scenario_119,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required submission contract." +us,scenario_119,federal_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model correctly identified the $764.23 EITC but misallocated the CTC, leaving only $583.72 refundable after claiming $3,816.28 was used nonrefundably. The refundable CTC is $2,089.77 after the correct nonrefundable-credit calculation." +us,scenario_119,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly treated $55,000 of wages as exceeding the two-child EITC limit and asserted that tax absorbs the full CTC. The correct derivation retains $764.23 of EITC and $2,089.77 of refundable CTC." +us,scenario_119,federal_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model applied married-filing-jointly EITC thresholds to a head-of-household filer and subtracted phaseout from the maximum as though phaseout began near $49,000, producing an impossible $6,400 EITC. The head-of-household calculation yields $764.23 of EITC, and refundable CTC is $2,089.77." +us,scenario_119,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly treated the $300,000 stock balance as disqualifying investment income; EITC uses investment income flows, and only $800 of taxable interest is listed. It therefore omitted the $764.23 EITC and also understated refundable CTC, which equals $2,089.77 rather than $565." us,scenario_119,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_119,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly subtracted the $6,389 employer-sponsored insurance premium from FICA wages as though it were a pre-tax employee salary reduction. The payroll-tax base remains $55,000, producing $3,410 of Social Security tax and $797.50 of Medicare tax." -us,scenario_119,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated the listed employer-sponsored insurance premium as a Section 125 pre-tax employee contribution and reduced FICA wages to $48,611. No such salary reduction is specified, so 7.65% applies to the full $55,000." -us,scenario_119,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model wrongly classified the $6,389 employer-sponsored insurance premium as an employee pre-tax payroll deduction. Using the full $55,000 FICA wage base yields $4,207.50, not $3,718.74." -us,scenario_119,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model incorrectly reduced taxable FICA wages by the employer-sponsored insurance premium despite no stated pre-tax employee salary reduction. Social Security and Medicare taxes apply to all $55,000 of wages." -us,scenario_119,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model identified the correct $55,000 wage base and FICA components but failed to apply and sum the statutory 6.2% Social Security and 1.45% Medicare rates. Those calculations produce $3,410 and $797.50, totaling $4,207.50 rather than $4,025." -us,scenario_119,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly deducted the $6,389 employer-sponsored insurance premium from FICA wages, then rounded the resulting component amounts. The applicable wage base is $55,000, yielding total employee FICA of $4,207.50." -us,scenario_119,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, violating the required output contract. The required computation is $3,410 of employee Social Security tax plus $797.50 of employee Medicare tax, totaling $4,207.50." -us,scenario_119,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly computed $3,410 of Social Security tax and $797.50 of Medicare tax but submitted $4,205 instead of their exact sum, $4,207.50. This is a final addition or transcription error that contradicts its own component calculations." -us,scenario_119,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model applied a four-person poverty guideline even though the household contains one head and two children. Under the three-person guideline, $55,800 is 204% of FPG, above the 185% reduced-price limit." -us,scenario_119,reduced_price_school_meals_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced the household's school-meal income from $55,800 to $46,180 by applying deductions that do not produce the income measure in the engine trace. The resulting school-meal ratio is 204% of FPG, not below 185%." -us,scenario_119,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model stated that household income was below a $47,767 threshold despite the listed $55,000 wages and $800 interest totaling $55,800. That income equals 204% of FPG and exceeds the 185% reduced-price limit." -us,scenario_119,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model placed the household between the free- and reduced-price income limits without computing the applicable ratio. The engine income of $55,800 is 204% of FPG, above the 185% ceiling, so the household is in the PAID tier." -us,scenario_119,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used $46,180 of MAGI instead of the $55,800 school-meal income measure, incorrectly lowering the ratio to 168% of FPG. The correct ratio is 204%, which exceeds the 185% reduced-price threshold." +us,scenario_119,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model incorrectly treated the $6,389 employer-sponsored insurance premium as a pre-tax employee deduction from FICA wages. Payroll taxes apply to the full $55,000, producing $3,410 of Social Security tax and $797.50 of Medicare tax." +us,scenario_119,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted the employer-sponsored insurance premium from gross wages even though the facts do not establish a pre-tax employee premium deduction. Applying 7.65% to the full $55,000 yields $4,207.50." +us,scenario_119,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA-taxable wages from $55,000 to $48,611 using the listed employer-sponsored insurance premium. The applicable payroll-tax base remains $55,000." +us,scenario_119,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model misclassified the $6,389 employer-sponsored insurance premium as a pre-tax reduction to FICA wages. Social Security and Medicare taxes are computed on the full $55,000, not $48,611." +us,scenario_119,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model identified Social Security and Medicare as the applicable taxes but did not correctly apply their combined 7.65% rate to $55,000. The stated wage base produces $4,207.50, not $4,025." +us,scenario_119,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly used $48,611 as FICA wages after subtracting the $6,389 employer-sponsored insurance premium. The full $55,000 is subject to the 6.2% Social Security and 1.45% Medicare rates." +us,scenario_119,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract." +us,scenario_119,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly computed $3,410 of Social Security tax and $797.50 of Medicare tax but then submitted $4,205 instead of their exact sum, $4,207.50. This is a final addition or transcription error." +us,scenario_119,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model used a four-person poverty guideline even though the household contains one head and two children. For the correct three-person household, $55,800 equals 204% of the guideline and exceeds the 185% reduced-price limit." +us,scenario_119,reduced_price_school_meals_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced the household's school-meal income from $55,800 to $46,180 by applying deductions that do not enter this school-meal income calculation. The resulting income is 204% of the guideline, not below 185%." +us,scenario_119,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model declared income below $47,767 despite the household's $55,000 of wages and $800 of interest. School-meal income is $55,800, which exceeds the 185% threshold and places the household in the PAID tier." +us,scenario_119,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model placed the household between the free- and reduced-price income limits without computing the school-meal FPG ratio. The correct ratio is 2.04, above the 1.85 reduced-price cutoff, and no categorical pathway overrides that result." +us,scenario_119,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly substituted $46,180 of MAGI for the $55,800 income used by the school-meal test. At $55,800, the household is at 204% of the guideline rather than 168%, so it exceeds the 185% limit." us,scenario_119,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly derived $39,469 of taxable income and about $2,012 of bracket tax, then abandoned that calculation and omitted the $152.85 nonrefundable Virginia EITC. Applying that credit yields $1,859.09." -us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $4,500 single standard deduction instead of the $8,750 head-of-household deduction and omitted the $1,560 dependent-care deduction and $2,790 of exemptions. It also failed to apply the $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction and omitted the $1,560 dependent-care deduction, leaving taxable income above $39,468.54. Its unspecified “small” credit did not apply the exact $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's own stated calculation produced about $2,116, but it inexplicably changed the result to $2,459. It also used an $8,500 standard deduction, omitted the $1,560 dependent-care deduction, and omitted the $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model calculated approximately $2,116 before credits but then increased the amount to $2,280 despite describing nonrefundable credits, which can only reduce liability. It also omitted the dependent-care deduction and exact $152.85 Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $9,000 standard deduction instead of $8,750, omitted the $1,560 Virginia dependent-care deduction, and concluded that no applicable nonrefundable credit existed. The applicable Virginia EITC reduces bracket tax by $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted both the $1,560 dependent-care deduction and $2,790 of personal exemptions from Virginia taxable income. It then invented a $220 adjustment and a child-care credit instead of applying the $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted only an estimated $8,500 standard deduction, omitting $2,790 of personal exemptions and the $1,560 dependent-care deduction. It substituted a $124.80 CDCC credit for the applicable $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model overstated the Virginia EITC as $660 by using an inflated federal EITC rather than the traced potential EITC of $764.23 and resulting $152.85 state credit. It also reduced tax with a child-care credit even though the trace treats the $1,560 expense as a taxable-income deduction." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $2,043 result does not reflect the traced $39,468.54 taxable income after the standard deduction, dependent-care deduction, and exemptions. It also fails to subtract the $152.85 nonrefundable Virginia EITC from $2,011.94 of bracket tax." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced AGI to $46,180 and used a $3,000 standard deduction instead of starting from $52,568.54 and subtracting $8,750. It also overstated the Virginia EITC as $386.60 instead of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $46,180 of AGI, an $8,500 standard deduction, and only $2,400 of exemptions, producing $35,280 rather than $39,468.54 of taxable income. It then applied a child-care credit instead of the $152.85 Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The submitted amount implies that the model did not use the traced combination of $52,568.54 AGI, $8,750 standard deduction, $1,560 dependent-care deduction, and $2,790 exemptions. It also failed to reduce the resulting $2,011.94 bracket tax by exactly $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model omitted the $2,790 personal exemptions and $1,560 dependent-care deduction from taxable income. More decisively, it calculated a $5,669.56 federal EITC despite this household's earnings being near the phaseout range; the traced potential EITC is $764.23, producing only a $152.85 Virginia credit." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model provided no computation supporting $803.75. The required derivation produces $39,468.54 of taxable income, $2,011.94 of bracket tax, and a $152.85 nonrefundable Virginia EITC, leaving $1,859.09." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model provided no rule or arithmetic supporting a residual liability of $180. The bracket calculation yields $2,011.94 and the applicable nonrefundable credit is only $152.85, so liability is $1,859.09." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model stated taxable income of about $45,219, but the listed deductions and exemptions reduce it to $39,468.54. It also overstated the Virginia EITC as $267 instead of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model named the correct deduction categories but its $1,483.02 result does not follow from them. Taxable income of $39,468.54 produces $2,011.94 under Virginia's brackets, followed by a $152.85 EITC reduction to $1,859.09." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model correctly reached approximately $2,012 after the standard deduction, exemptions, and dependent-care deduction, but incorrectly declared that no nonrefundable low-income credit applied. The $152.85 nonrefundable Virginia EITC reduces that amount to $1,859.09." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model invented two $500 nonrefundable child credits, reducing liability by $1,000. The applicable traced nonrefundable credit is the $152.85 Virginia EITC, not child credits." -us,scenario_119,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model gave no taxable-income arithmetic and omitted the applicable credit from its explanation. The listed deductions yield $39,468.54 of taxable income and $2,011.94 of bracket tax, which the $152.85 Virginia EITC reduces to $1,859.09." -us,scenario_119,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction instead of $8,750 and omitted the $1,560 dependent-care deduction, overstating taxable income as $41,279. It also omitted the $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used $35,330 of taxable income without deriving it from the household facts; the traced deductions yield $39,468.54. It also stopped at bracket tax and omitted the $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract." -us,scenario_119,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction instead of $8,750 and omitted the $1,560 dependent-care deduction. It also incorrectly stated that no nonrefundable credit applied, omitting the $152.85 Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used incorrect Virginia brackets, applying 3% through $17,000 and 5% above $17,000 rather than the traced schedule culminating in 5.75% above $17,000. It also used the wrong standard deduction and exemption rules, and its final $2,867 contradicts its own stated pre-exemption tax of about $2,120." -us,scenario_119,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model improperly imputed employer-paid health-insurance value into AGI and treated this head-of-household filer as married filing jointly with an $18,000 deduction. Virginia starts from $52,568.54 of AGI here and applies the $8,750 head-of-household deduction, $1,560 dependent-care deduction, $2,790 exemptions, and $152.85 EITC." -us,scenario_119,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model treated Virginia’s 20% nonrefundable EITC as a refundable credit in 2026. Virginia’s temporary refundable election had expired, so multiplying the federal EITC by 20% cannot produce state_refundable_credits." -us,scenario_119,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model extended Virginia’s temporary 15% refundable EITC election beyond its tax-year-2025 expiration. No refundable Virginia EITC applies in 2026, so the submitted $332.46 belongs to an inapplicable eligibility pathway." -us,scenario_119,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model classified a 20%-of-federal-EITC calculation as refundable for Virginia in 2026. The 20% Virginia EITC pathway is nonrefundable, while the temporary refundable election ended after 2025." -us,scenario_119,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,The model applied a nonexistent 2026 refundable Virginia EITC equal to 20% of the federal credit. Virginia’s continuing 20% EITC is nonrefundable and does not enter state_refundable_credits. -us,scenario_119,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, violating the required output contract. The required 2026 amount is $0 because no Virginia refundable credit applies." -us,scenario_119,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model incorrectly placed 20% of its computed federal EITC into Virginia refundable credits. That 20% state EITC pathway is nonrefundable, and Virginia’s temporary refundable election does not apply after 2025." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model derived taxable income near $200,500 but then applied the 2026 single-filer brackets incorrectly, producing only $30,238 instead of regular tax near $40,000. It also omitted the $163.40 net investment income tax and used incorrect AGI and deduction inputs." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated all Social Security as taxable, used an invented $30,000 standard deduction, applied head-of-household rates to a single filer, and subtracted an unlisted $11,297 of withholding from tax liability. Withholding is a payment, not a nonrefundable credit, and does not reduce this output." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used $234,385 of AGI and a $10,000 SALT cap instead of the traced $232,635.12 AGI and $13,907.78 SALT deduction, leaving taxable income too high. It then failed to carry its own stated bracket calculation of about $40,823 into its submitted $38,740 and omitted the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model included the $1,750 state tax refund in AGI and limited SALT to $10,000, whereas the computation uses AGI of $232,635.12 and a $13,907.78 SALT deduction. Those errors produced $202,030 of taxable income instead of $196,918.41, and it also omitted the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model counted only the listed $2,012 real-estate tax rather than the full $13,907.78 deductible SALT amount and used $3,744 rather than $2,808 of deductible mortgage interest. This understated itemized deductions by nearly $10,000, raised taxable income to about $208,465, and omitted the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly assumed a wholesale TCJA sunset for 2026, restored pre-TCJA rates, personal exemptions, and miscellaneous employee-expense deductions, and repeatedly changed its bracket estimates without reconciling the arithmetic. The applicable computation instead uses $196,918.41 of taxable income under 2026 current-law brackets plus $163.40 of net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Although the model got close to the traced AGI, it used only $24,984 of itemized deductions instead of $35,716.73 by failing to include the full deductible SALT amount. It then drastically undercalculated bracket tax on its own $207,651 taxable-income figure and omitted the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $7,429 employer-sponsored insurance premium from wages and restored a personal exemption and miscellaneous itemized deduction. It also used the wrong itemized-deduction components and calculated net investment income tax on only $1,853 instead of the traced $4,300 base." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete post-sunset framework, including uncapped SALT, a miscellaneous employee-expense deduction, and an IRA adjustment, rather than the traced 2026 computation. It also omitted the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied no usable computation and returned $39,591. The correct derivation produces $39,858.42 of main income tax and then adds $163.40 of net investment income tax, so its answer reflects an incomplete or inaccurate tax calculation." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $7,429 employer-sponsored insurance premium from wages, driving AGI down to about $226,956 instead of $232,635.12. It also used the wrong itemized-deduction regime and calculated only about $87 of net investment income tax instead of $163.40." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model deducted employer insurance premiums from wages and restored both a personal exemption and the obsolete miscellaneous deduction for unreimbursed employee expenses. Those errors reduced taxable income to $180,066 instead of $196,918.41, while its $70 net investment income tax also understated the traced $163.40." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted employer-sponsored insurance premiums from wages, producing AGI of $226,985 instead of $232,635.12. It also used only $32,972 of itemized deductions and omitted the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included only $2,012 of SALT rather than the traced $13,907.78, so it understated itemized deductions by $10,732.73 and overstated taxable income at $209,430. It also omitted the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model gave no calculation supporting $38,141 and incorrectly described itemized deductions as being limited by the standard deduction. Itemized deductions of $35,716.73 exceed and replace the $18,150 standard deduction, yielding $39,858.42 of main tax plus $163.40 of net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model conflated deductions such as mortgage interest and SALT with nonrefundable credits and gave no reproducible taxable-income calculation. The applicable deductions produce taxable income of $196,918.41, followed by $39,858.42 of main tax and $163.40 of net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly imposed a post-TCJA-sunset regime, including a personal exemption and obsolete deduction rules, and used only about $70 of net investment income tax. The current-law calculation has no restored personal exemption and adds $163.40 of net investment income tax to $39,858.42 of main tax." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model invoked a 2026 charitable-contribution floor and arrived at $210,286.16 of taxable income rather than the traced $196,918.41. The deductible charitable amount is $19,000.94 within total itemized deductions of $35,716.73, and the final computation also includes $163.40 of net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model started from approximately $234,414 of AGI instead of $232,635.12 and used the wrong itemized-deduction amounts. It also added only about $70 of net investment income tax rather than $163.40." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption under a post-TCJA-sunset framework and consequently used the wrong taxable income and rate schedule. The applicable taxable income is $196,918.41 with no restored personal exemption, and the result includes $163.40 of net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model provided no reproducible amounts for AGI, itemized deductions, taxable income, bracket tax, or net investment income tax. The traced sequence yields $39,858.42 of main tax and requires adding $163.40, which its $39,250 answer fails to capture." -us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly restored pre-TCJA rates, a personal exemption, and the miscellaneous deduction for unreimbursed employee expenses. It also omitted the $163.40 net investment income tax and used $234,414 rather than $232,635.12 of AGI." -us,scenario_120,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an incorrect post-TCJA-expiration rate and deduction regime, including a miscellaneous employee-expense deduction and an estimated SALT amount inconsistent with the traced $13,907.78. It also omitted the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so it failed the required output contract." -us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so it failed the required output contract." -us,scenario_120,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model returned zero despite $196,918.41 of taxable income and provided no computation. The applicable calculation produces $39,858.42 of main income tax plus $163.40 of net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $617 pre-tax 401(k) reduction and otherwise mis-added gross-income components to obtain $233,031, then used the standard deduction instead of the larger $35,716.73 itemized deduction. It also omitted the $163.40 net investment income tax." +us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model correctly reconstructed the $39,469 taxable income and roughly $2,012 bracket tax, then discarded that calculation and submitted an unsupported $2,296. It also failed to subtract the $152.85 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model used a $4,500 single standard deduction instead of the $8,750 head-of-household deduction and omitted both the $1,560 dependent-care deduction and $2,790 of personal exemptions. It also did not apply the $152.85 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model used an $8,500 standard deduction instead of $8,750 and omitted the $1,560 dependent-care deduction. It then estimated an unspecified credit rather than applying the exact $152.85 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model omitted the $1,560 dependent-care deduction, used an $8,500 rather than $8,750 standard deduction, and did not subtract the $152.85 Virginia EITC. Its submitted $2,459 also contradicts its own calculated $2,116 bracket tax." +us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model omitted the dependent-care deduction, understated the standard deduction by $250, and failed to apply the nonrefundable Virginia EITC. Its claimed upward 'small adjustments' from $2,116 to $2,280 are incompatible with an after-nonrefundable-credit output." +us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model used a $9,000 standard deduction, omitted the $1,560 dependent-care deduction, and concluded that no nonrefundable credit applied. The computation requires an $8,750 standard deduction and subtraction of the $152.85 Virginia EITC from $2,011.94." +us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model omitted the $2,790 personal exemptions and $1,560 dependent-care deduction when constructing taxable income. It then invented a $220 adjustment and an estimated child-care credit instead of applying the $152.85 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model omitted the $2,790 personal exemptions and $1,560 dependent-care deduction and used an $8,500 standard deduction instead of $8,750. It also substituted a $124.80 child-care credit for the $152.85 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model overstated the Virginia EITC as $660 instead of $152.85 and also subtracted a $78 child-care credit rather than treating the $1,560 expense as a taxable-income deduction. Its stated $34,780 taxable income is below the traced $39,468.54." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The terse answer implies that the model did not complete the traced sequence of $39,468.54 taxable income, $2,011.94 bracket tax, and $152.85 nonrefundable EITC. That sequence yields $1,859.09, not $2,043." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated Virginia AGI as $46,180 instead of $52,568.54 and used a $3,000 standard deduction instead of $8,750. It also overstated the nonrefundable Virginia EITC as $386.60 rather than $152.85." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $46,180 and personal exemptions as $2,400 rather than using $52,568.54 and $2,790. It treated dependent care as a $62.40 credit instead of deducting the full $1,560 from taxable income and then applying the $152.85 Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The submitted amount does not follow the required traced calculation: deductions and exemptions produce $39,468.54 of taxable income, bracket tax is $2,011.94, and the nonrefundable Virginia EITC is $152.85. The model's unexplained credit calculation reduced the liability to the wrong amount." +us,scenario_119,state_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model omitted the $2,790 personal exemptions and $1,560 dependent-care deduction from taxable income. It then calculated a $5,669.56 federal EITC and $1,133.91 Virginia credit, whereas the applicable potential EITC is $764.23 and the nonrefundable Virginia EITC is $152.85." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The unexplained $803.75 result skips the traced Virginia calculation of $39,468.54 taxable income, $2,011.94 bracket tax, and a $152.85 nonrefundable EITC. It reflects an excessive unitemized reduction of the state liability." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The unexplained $180 residual applies far more nonrefundable credit than the traced $152.85 Virginia EITC. Correct deductions produce $2,011.94 before credits and $1,859.09 after that credit." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model calculated taxable income as about $45,219 instead of $39,468.54, so it failed to apply the full $8,750 standard deduction, $2,790 exemptions, and $1,560 dependent-care deduction. It also used an estimated $267 EITC rather than $152.85." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model named the correct deduction categories but its $1,483.02 result does not follow their traced amounts and the Virginia brackets. Those deductions yield $2,011.94 before credits, followed by a $152.85 nonrefundable EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model correctly reached approximately $2,012 after the standard deduction, exemptions, and dependent-care deduction, but stopped before the required nonrefundable-credit step. Subtracting the $152.85 Virginia EITC yields $1,859.09." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model invented two $500 nonrefundable child credits that are not part of this Virginia calculation. The applicable nonrefundable reduction is the single $152.85 Virginia EITC from the $2,011.94 bracket tax." +us,scenario_119,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model gave no usable derivation and did not identify the $1,560 dependent-care deduction or $152.85 nonrefundable Virginia EITC. Applying all traced deductions and the credit yields $1,859.09 rather than $1,874." +us,scenario_119,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model used an $8,500 standard deduction rather than $8,750 and omitted the $1,560 dependent-care deduction. It also stopped at bracket tax without subtracting the $152.85 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used $35,330 of taxable income rather than the traced $39,468.54, so its deductions exceeded the allowed $8,750 standard deduction, $1,560 dependent-care deduction, and $2,790 exemptions. It also omitted the separate $152.85 nonrefundable Virginia EITC step." +us,scenario_119,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model used taxable income of about $41,779 instead of $39,468.54, omitting or misstating the dependent-care and standard deductions. It also used a roughly $238 Virginia EITC instead of the traced $152.85 credit." +us,scenario_119,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. +us,scenario_119,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model used an $8,500 standard deduction instead of $8,750, omitted the $1,560 dependent-care deduction, and incorrectly concluded that no nonrefundable state credit applied. The $152.85 Virginia EITC must be subtracted from $2,011.94." +us,scenario_119,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used incorrect Virginia brackets, applying 3% through $17,000 and 5% above $17,000 instead of the 2%, 3%, 5%, and 5.75% schedule. It also used the wrong standard deduction and exemptions, omitted the dependent-care deduction and EITC, and submitted $2,867 despite its own $2,120 pre-exemption calculation." +us,scenario_119,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,The model imputed employer health-insurance income that was not listed and treated the filer as married filing jointly despite the specified head-of-household status. It therefore used the wrong AGI and standard deduction and never reproduced the $152.85 nonrefundable Virginia EITC. +us,scenario_119,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model overstated taxable income as $50,200 by failing to apply the traced $8,750 standard deduction, $1,560 dependent-care deduction, and $2,790 personal exemptions to $52,568.54 of AGI. It also invented a $112 low-income credit instead of applying the $152.85 Virginia EITC." +us,scenario_119,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly applied a Virginia refundable EITC in 2026 and used a 20% rate. No refundable Virginia EITC applies for this tax year, so the computed $1,431 federal EITC produces $0 of state refundable credits." +us,scenario_119,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model extended Virginia's 15%-of-federal-EITC refundable provision into 2026. That refundable provision does not apply in the benchmark year, so the federal EITC generates no Virginia refundable credit." +us,scenario_119,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model incorrectly treated 20% of its estimated $785 federal EITC as a refundable Virginia credit. Virginia provides no applicable refundable EITC in 2026, so this multiplication should not occur." +us,scenario_119,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied a 20% refundable Virginia EITC to its $1,300 federal EITC estimate. No such refundable credit applies for Virginia in tax year 2026, leaving state refundable credits at $0." +us,scenario_119,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model omitted the required state_refundable_credits output entirely. The required value was $0 because no refundable Virginia credit applies to this household in 2026. +us,scenario_119,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model incorrectly calculated a Virginia refundable credit as 20% of a $764.23 federal EITC. Virginia's modeled 2026 rules provide no applicable refundable EITC, so the correct state contribution from the federal EITC is $0." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model derived taxable income near $200,500 but then computed only $30,238 of tax, which is not the tax produced by the 2026 single-filer brackets. It also used a $10,000 SALT cap instead of the $13,907.78 deductible amount and failed to add $163.40 of net investment income tax." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated all Social Security as taxable, used an invented $30,000 standard deduction, and omitted the much larger allowable itemized deduction. It then improperly subtracted $11,297 of estimated withholding even though withholding does not reduce federal income tax before refundable credits." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model estimated AGI and taxable income near the correct ranges but its bracket arithmetic produced $40,823 and then replaced that result with an unexplained $38,740. It also imposed a $10,000 SALT cap, deducted the full stated charity rather than $19,000.94, and omitted the $163.40 net investment income tax." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model included the $1,750 state tax refund in AGI and limited SALT to $10,000, producing $202,030 of taxable income instead of $196,918.41. It also omitted the $163.40 net investment income tax." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model counted only $2,012 of SALT rather than the $13,907.78 deduction, used the full $3,744 mortgage interest instead of the $2,808 debt-limited amount, and did not apply the charitable-contribution floor. These errors raised taxable income to about $208,465 instead of $196,918.41, and it also omitted NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly assumed a wholesale TCJA sunset for 2026, restoring pre-TCJA rates, personal exemptions, and miscellaneous employee-expense deductions. The applicable 2026 computation retains the current single-filer rate structure, uses $35,716.73 of itemized deductions without a personal exemption, and adds $163.40 of NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Although the model reached approximately the correct AGI, it used only about $24,984 of itemized deductions by omitting the deductible state-income-tax component of the $13,907.78 SALT deduction and failing to apply the correct charitable amount. It then drastically undercomputed the bracket tax on its own stated $207,651 taxable income." +us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $7,429 of employer-sponsored insurance premiums from wages and restored a personal exemption and miscellaneous employee-expense deduction. It also used full mortgage interest and the wrong SALT and NIIT bases instead of taxable income of $196,918.41 and NIIT of $163.40." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied supposed post-TCJA-sunset provisions, including uncapped SALT and a miscellaneous itemized deduction for unreimbursed employee expenses. It also used full mortgage interest and full charitable contributions, rather than the traced deductions of $13,907.78, $2,808, and $19,000.94." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's unsupported $39,591 estimate does not implement the traced deduction calculation. Correctly reducing $232,635.12 of AGI by $35,716.73 of itemized deductions yields $196,918.41 of taxable income, followed by $39,858.42 of regular tax and $163.40 of NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the $7,429 employer-sponsored insurance premium from wages and included restored miscellaneous itemized deductions. This understated AGI and used the wrong deduction base; the correct NIIT is also $163.40 rather than roughly $87." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by employer health premiums, restored a personal exemption, and deducted unreimbursed employee expenses above a 2% floor. It also used full mortgage interest and an incorrect SALT amount instead of the traced $35,716.73 total itemized deduction." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted employer-sponsored health premiums from wages, reducing AGI to $226,985 instead of $232,635.12. It also used a $10,000 SALT cap and omitted the $163.40 net investment income tax." +us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model counted only the $2,012 property tax as SALT and therefore used $24,984 of itemized deductions instead of $35,716.73. It also included the state tax refund in AGI, failed to apply the charitable floor, and omitted NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model supplied no computation supporting $38,141 and incorrectly described itemized deductions as limited by the standard deduction. Itemized deductions exceed the standard deduction and reduce $232,635.12 of AGI to $196,918.41 before regular tax and $163.40 of NIIT are calculated." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model conflated deductions such as mortgage interest and SALT with nonrefundable credits and provided no valid taxable-income derivation. The correct computation uses those items within a $35,716.73 itemized deduction, then applies the brackets and $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly applied post-TCJA-sunset rules, restored a personal exemption, and used pre-TCJA rates. It also overstated itemized deductions and calculated only about $70 of NIIT instead of $163.40." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model reached $210,286.16 of taxable income instead of $196,918.41, so it failed to include the full traced $35,716.73 itemized deduction. Its result also omits the separately computed $163.40 net investment income tax." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used the wrong AGI and itemized-deduction amounts, including an incorrect treatment of the state tax refund and SALT deduction. It also added only about $70 of NIIT when all $4,300 of net investment income produces $163.40." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and applied post-TCJA-sunset tax rules. The applicable calculation has no personal exemption, produces $196,918.41 of taxable income under the 2026 single-filer brackets, and adds $163.40 of NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's unsupported $39,250 estimate does not reproduce the traced itemized deductions or NIIT. The correct steps yield $39,858.42 of regular tax on $196,918.41 of taxable income plus $163.40 of NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model wrongly restored pre-TCJA rates, a personal exemption, and the miscellaneous itemized deduction for employee expenses. It also used full mortgage interest and full charitable contributions rather than the applicable debt limitation and charitable floor." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied post-TCJA-expiration rates and deducted unreimbursed employee expenses under restored miscellaneous-deduction rules. Its estimated SALT, full mortgage interest, and full charitable deduction also differ from the traced $13,907.78, $2,808, and $19,000.94 amounts." +us,scenario_120,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model restored a miscellaneous itemized deduction for employee expenses and used incorrect AGI, SALT, and charitable-deduction amounts. Although it identified $163 of NIIT, its wrong deductions produced taxable income of about $194,235 instead of $196,918.41." +us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for the requested variable. +us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for the requested variable. +us,scenario_120,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The zero answer omits the entire federal tax computation. This filer has $196,918.41 of taxable income, generating $39,858.42 of regular income tax plus $163.40 of NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used the standard deduction even though the $35,716.73 itemized deduction is larger, and it omitted the pre-tax 401(k) reduction and other traced AGI adjustments. It also failed to add $163.40 of NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model treated the explicitly single filer as married filing separately and incorrectly included the tax-exempt pension in taxable income. It also combined the standard deduction with charitable itemization, invented a retirement-income credit, omitted the mortgage and SALT deduction calculation, and failed to reconcile its stated $45,287 less $750 with its submitted $48,557." us,scenario_120,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_120,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no head_medicare_eligible output, violating the required structured-output contract." -us,scenario_120,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,The model returned 0 despite the head being age 76. It failed to apply age-based Medicare eligibility beginning at age 65; the head's high income and employer-sponsored insurance do not eliminate eligibility. +us,scenario_120,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model did not submit a value for head_medicare_eligible, violating the required output contract. The head's age of 76 yields eligibility under the age-65 Medicare rule." +us,scenario_120,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model returned 0 without applying Medicare's age-based eligibility rule. At age 76, the head exceeds the age-65 threshold, so the eligibility value is 1 regardless of the household's income, assets, or employer-sponsored coverage." us,scenario_120,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_120,payroll_tax,claude-fable-5,llm_error,other,False,"The model derived every component correctly and explicitly totaled them to $13,496.16, but submitted the unrelated value $13,736.63. Its numeric output contradicts its completed calculation." -us,scenario_120,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,The model replaced Connecticut's 0.5% paid-leave contribution of $827.98 with a fabricated negative $48.79 “net from itemizations.” Itemized deductions do not reduce or offset this employee payroll contribution. -us,scenario_120,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model omitted Connecticut's mandatory employee paid-leave contribution. Adding the $827.98 contribution to its correctly calculated federal FICA of $12,668.17 yields $13,496.16." -us,scenario_120,payroll_tax,claude-opus-5,llm_error,other,False,"The model correctly identified $10,267.01 of Social Security tax, $2,401.16 of Medicare tax, and about $827.98 of Connecticut paid-leave tax, but then submitted $10,680 instead of summing those components. Their sum is $13,496.16." -us,scenario_120,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly stated that Connecticut has no mandatory employee-side state payroll tax. Connecticut's employee paid-leave contribution adds $827.98 to the $12,668.17 federal FICA amount." -us,scenario_120,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $7,429 employer-sponsored insurance premium from wages for Social Security, Medicare, and Connecticut paid-leave tax. The trace uses the full $165,597 of taxable earnings, producing components of $10,267.01, $2,401.16, and $827.98." -us,scenario_120,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $14,197 exceeds the taxes produced by the stated 6.2% Social Security and 1.45% Medicare rates and the applicable Connecticut contribution. Wages are below the $200,000 Additional Medicare Tax threshold, so the correct components total $13,496.16 without any additional Medicare tax." -us,scenario_120,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced payroll-tax wages by the $7,429 employer-sponsored insurance premium and applied all three taxes to $158,168. The computation trace applies Social Security, Medicare, and Connecticut paid leave to the full $165,597, yielding $13,496.16." -us,scenario_120,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model subtracted the $7,429 employer-sponsored insurance premium from the base for Social Security, Medicare, and Connecticut paid leave. All three components use the full $165,597 in this case and total $13,496.16." -us,scenario_120,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model used $158,168 after subtracting the employer-sponsored insurance premium as the payroll-tax base. The applicable taxable earnings are $165,597, producing $10,267.01 of Social Security tax, $2,401.16 of Medicare tax, and $827.98 of Connecticut paid-leave tax." -us,scenario_120,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly deducted the traditional 401(k), Roth 401(k), and employer-sponsored insurance amounts when constructing FICA wages. The trace taxes the full $165,597 and also includes Connecticut's $827.98 employee paid-leave contribution, which the model omitted." -us,scenario_120,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly asserted that Connecticut has no mandatory employee state payroll tax and also failed to produce the federal FICA amount implied by its own stated method. Social Security and Medicare total $12,668.17, and Connecticut paid leave adds $827.98 for a total of $13,496.16." -us,scenario_120,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model stated that both 6.2% Social Security and 1.45% Medicare apply to all $165,597 but submitted $6,180, which is inconsistent with those rates. Federal FICA alone is $12,668.17, and the Connecticut paid-leave contribution raises the total to $13,496.16." -us,scenario_120,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"The model overstated Connecticut's employee paid-leave withholding as $922.50 instead of $827.98. Combining the correct state contribution with its correct Social Security and Medicare amounts gives $13,496.16." -us,scenario_120,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $617 traditional 401(k) deferral from Social Security and Medicare wages even though 401(k) elective deferrals remain subject to FICA. It also omitted Connecticut's $827.98 employee paid-leave contribution; the full-wage components total $13,496.16." -us,scenario_120,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"The model calculated only federal Social Security and Medicare taxes on gross wages. It omitted Connecticut's mandatory $827.98 employee paid-leave contribution, which raises the total from $12,668.17 to $13,496.16." -us,scenario_120,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly stated that Connecticut has no employee state payroll tax. The Connecticut paid-leave contribution is $827.98 and must be added to federal FICA of $12,668.17." -us,scenario_120,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"The model stopped after Social Security and Medicare and incorrectly concluded that there were no other employee payroll taxes. Connecticut's mandatory employee paid-leave contribution adds $827.98, producing $13,496.16." -us,scenario_120,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll-tax output. The required numeric value was therefore missing. -us,scenario_120,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no payroll-tax output. The required numeric value was therefore missing. -us,scenario_120,payroll_tax,minimax-m3,llm_error,other,False,"The model returned zero despite $165,597 of employee wages. Those wages generate $10,267.01 of Social Security tax, $2,401.16 of Medicare tax, and $827.98 of Connecticut paid-leave tax." -us,scenario_120,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly stated that Connecticut has no mandatory employee state payroll tax and therefore included only federal FICA. Connecticut's $827.98 employee paid-leave contribution raises its $12,668.17 subtotal to $13,496.16." +us,scenario_120,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly derived all three components and explicitly obtained $13,496.16, but submitted the unrelated value $13,736.63. Its final numeric output failed to carry forward its own calculation." +us,scenario_120,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model replaced Connecticut's required 0.5% paid-leave contribution of $827.98 with an invented negative $48.79 adjustment tied to itemizations. Itemized deductions do not offset employee payroll taxes, so the three payroll-tax components total $13,496.16." +us,scenario_120,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model omitted Connecticut's mandatory employee paid-leave contribution. Adding the $827.98 state contribution to its correctly computed $12,668.17 of federal FICA yields $13,496.16." +us,scenario_120,payroll_tax,claude-opus-5,llm_error,other,False,"The model correctly identified approximately $12,668 of federal FICA and $828 of Connecticut paid-leave tax, then replaced their approximately $13,496 sum with $10,680 without a valid computation. The stated components sum to $13,496.16." +us,scenario_120,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly asserted that Connecticut has no mandatory employee-side state payroll tax. Connecticut's paid-leave contribution adds $827.98 to federal FICA, producing $13,496.16." +us,scenario_120,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $7,429 employer-sponsored insurance premium from wages and applied every payroll-tax rate to $158,168. The computation trace uses the full $165,597 as taxable earnings for Social Security, Medicare, and Connecticut paid leave, yielding $13,496.16." +us,scenario_120,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model's $14,197 output applies payroll tax beyond the stated standard Social Security and Medicare calculation even though $165,597 is below the $200,000 Additional Medicare Tax threshold. The applicable components are $10,267.01 of Social Security, $2,401.16 of Medicare, and $827.98 of Connecticut paid leave, totaling $13,496.16." +us,scenario_120,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly reduced payroll-taxable wages from $165,597 to $158,168 by subtracting the employer-sponsored insurance premium. Applying Social Security, Medicare, and Connecticut paid-leave contributions to the full traced earnings produces $13,496.16." +us,scenario_120,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $7,429 employer-sponsored insurance premium from the base for Social Security, Medicare, and Connecticut paid leave. All three traced components use $165,597 of earnings and total $13,496.16." +us,scenario_120,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model applied all payroll-tax rates to $158,168 after an unsupported reduction for insurance premiums. The full $165,597 is the taxable-earnings base for the traced Social Security, Medicare, and Connecticut paid-leave amounts." +us,scenario_120,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model improperly removed the traditional 401(k), Roth 401(k), and employer-sponsored insurance amounts from FICA wages. These entries do not produce the reduced payroll-tax base it used; the trace applies the taxes to $165,597 and also includes $827.98 of Connecticut paid-leave contributions." +us,scenario_120,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly excluded Connecticut's mandatory employee paid-leave tax and also failed to compute federal FICA consistently from the full wages. Social Security and Medicare alone equal $12,668.17, and the $827.98 Connecticut contribution raises the total to $13,496.16." +us,scenario_120,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model stated that 6.2% Social Security and 1.45% Medicare apply to all $165,597 but submitted $6,180, which is arithmetically inconsistent with those rates. Federal FICA is $12,668.17, and Connecticut paid leave adds $827.98." +us,scenario_120,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"The model overstated Connecticut paid-family-leave withholding as $922.50. The traced 0.5% contribution on $165,597 is $827.98, which combines with $12,668.17 of federal FICA for $13,496.16." +us,scenario_120,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $617 traditional 401(k) deferral from FICA wages. It also omitted Connecticut's $827.98 employee paid-leave contribution; the trace taxes the full $165,597 and totals $13,496.16." +us,scenario_120,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"The model calculated only federal Social Security and Medicare taxes. It omitted Connecticut's mandatory $827.98 employee paid-leave contribution, which raises the result from $12,668.17 to $13,496.16." +us,scenario_120,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly stated that Connecticut has no employee state payroll tax. Connecticut's paid-leave contribution is $827.98, so it must be added to the $12,668.17 federal FICA amount." +us,scenario_120,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"The model omitted Connecticut's mandatory employee paid-leave contribution when it concluded there were no other employee payroll taxes. That state component contributes $827.98 and brings total payroll tax to $13,496.16." +us,scenario_120,payroll_tax,inkling,llm_error,state_local_rule,False,The model incorrectly asserted that Connecticut has no mandatory employee state payroll taxes. The Connecticut paid-leave contribution adds $827.98 to the correctly estimated federal FICA components. +us,scenario_120,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so the required output was missing." +us,scenario_120,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so the required output was missing." +us,scenario_120,payroll_tax,minimax-m3,llm_error,other,False,"The model returned zero despite $165,597 of employee wages. Those wages generate $10,267.01 of Social Security tax, $2,401.16 of Medicare tax, and $827.98 of Connecticut paid-leave contributions." +us,scenario_120,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model correctly calculated federal Social Security and Medicare taxes but incorrectly stated that Connecticut has no mandatory employee state payroll tax. Adding the $827.98 paid-leave contribution to $12,668.17 produces $13,496.16." +us,scenario_120,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model used an incorrect $9,655 Social Security cap and omitted both the regular 1.45% Medicare tax and Connecticut paid-leave contribution. Because $165,597 is fully within the traced Social Security base, Social Security is $10,267.01, with another $2,401.16 of Medicare and $827.98 of state payroll tax." us,scenario_120,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,snap,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,ssi,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model subtracted an estimated $35,615 of taxable Social Security and the $1,750 state refund while adding tax-exempt interest, instead of applying Connecticut's traced $25,140 Social Security subtraction to federal AGI of $232,635.12. That wrong Connecticut income base prevented the rate and $430 recapture calculation from yielding $11,917.18." -us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented a Connecticut deduction for federal income tax, imported federal itemized deductions, and asserted an over-65 deduction that eliminates liability. Connecticut instead taxes $207,495.12 after the specific Social Security adjustment, and those claimed deductions do not reduce the tax to zero." -us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model denied any Connecticut Social Security subtraction and taxed approximately $236,883. The applicable subtraction is $25,140 from federal AGI of $232,635.12, leaving $207,495.12 before applying the brackets and $430 of recaptures." -us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model approximated the income base and used an incorrect bracket schedule, including 5% rather than 4.5% for the second bracket and no precise recapture computation. Applying the Connecticut schedule and the $250 and $180 recaptures to $207,495.12 yields $11,917.18." -us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used obsolete rates topping out at 5.5% on an estimated $198,800 base. Connecticut taxable income is $207,495.12, reaches the 6.5% bracket, and receives $430 in low- and middle-rate recaptures." -us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated high income as eliminating the Connecticut Social Security subtraction and therefore taxed $234,414. Connecticut applies a $25,140 Social Security adjustment, reducing the traced $232,635.12 federal AGI to $207,495.12 before the rate and recapture calculations." -us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model deducted Roth contributions, mishandled the state refund and municipal interest, and described Social Security as largely exempt despite the stated high income. The traced computation instead starts at $232,635.12, subtracts exactly $25,140 for Social Security, and taxes $207,495.12 under the Connecticut schedule." -us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model created a $1,549.11 personal tax credit after reducing Connecticut AGI to $191,370. At this income no personal credit reduces liability; the correct income base is $207,495.12 and the tax remains $11,917.18 after the credit calculation." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted only 25% of federally taxable Social Security and separately removed the state refund, producing $223,731. Connecticut's applicable Social Security subtraction is $25,140, which reduces federal AGI of $232,635.12 to $207,495.12 before the two recaptures." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $7,800 estimate omits the traced $207,495.12 Connecticut taxable-income base and its graduated-rate computation. Applying the schedule plus $250 and $180 recaptures produces $11,917.18." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model explicitly denied a Social Security exemption because of high income and therefore retained too much income in the Connecticut base. Connecticut subtracts $25,140 of Social Security, leaving $207,495.12 rather than taxing federal AGI without that adjustment." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used federal AGI of $225,235 and a $26,711 Social Security exemption, producing $198,524. The traced inputs instead produce federal AGI of $232,635.12 and a $25,140 subtraction, so the Connecticut base is $207,495.12." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model placed Connecticut taxable income at $212,320 rather than $207,495.12. It therefore applied the brackets to an overstated base instead of subtracting exactly $25,140 from federal AGI of $232,635.12 and then adding the two recaptures." -us,scenario_120,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model fully exempted Social Security, invented a $15,000 standard deduction, and applied a $200 property-tax credit. Connecticut instead uses a $25,140 Social Security subtraction, no such standard deduction in this computation, and no personal credit reduction, leaving $207,495.12 subject to tax." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The $4,414 answer implies a substantially understated Connecticut income base or unsupported deductions and credits. The traced base is $207,495.12, and applying the graduated schedule with $430 of recaptures and no credit reduction yields $11,917.18." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked itemized deductions and retirement contributions without identifying Connecticut's actual income adjustment. Connecticut taxes $207,495.12 after the $25,140 Social Security subtraction, and no nonrefundable credit reduces the resulting $11,917.18." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model came close on the income base but used about $208,805 and estimated the recaptures as $500 plus $180. The traced base is $207,495.12, and the applicable recaptures are exactly $250 and $180, producing $11,917.18." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model taxed approximately $235,031 rather than applying the Connecticut Social Security subtraction. Federal AGI of $232,635.12 is reduced by $25,140 to $207,495.12 before the brackets and recaptures are applied." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used approximately $234,000 as Connecticut AGI and consequently overstated tax. Connecticut subtracts $25,140 of Social Security from federal AGI of $232,635.12, so the taxable base is $207,495.12." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that high income eliminates all Connecticut Social Security relief and used roughly $234,414 as Connecticut AGI. The applicable Social Security subtraction is $25,140, and the resulting $207,495.12 base plus the two recaptures produces $11,917.18." -us,scenario_120,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The unsupported $7,850 estimate implies an understated taxable-income base or nonexistent deductions. Connecticut taxable income is $207,495.12 after the specific Social Security subtraction, and the rate schedule plus $430 of recaptures yields $11,917.18." -us,scenario_120,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model removed all federally taxable Social Security and the state refund, reducing Connecticut AGI to $197,049. Connecticut instead subtracts exactly $25,140 from federal AGI of $232,635.12, leaving $207,495.12 and requiring the additional recapture calculations." -us,scenario_120,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an understated $201,651 base and obsolete 3% and 5% lower brackets. The applicable base is $207,495.12, the lower rates are 2% and 4.5%, and $250 plus $180 of recaptures must be included." -us,scenario_120,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required structured-output contract rather than completing the Connecticut calculation. -us,scenario_120,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required structured-output contract rather than completing the Connecticut calculation. -us,scenario_120,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer omits the positive Connecticut tax on $207,495.12 of taxable income. The graduated schedule and $430 of recaptures produce $11,917.18, with no personal credit reducing it." -us,scenario_120,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model fully exempted Social Security, subtracted an already excluded tax-exempt pension, and retained a $15,000 personal exemption, reducing the base to $136,825. Connecticut instead applies only the $25,140 Social Security subtraction to federal AGI of $232,635.12, and the personal exemption provides no reduction at this income." +us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model constructed the wrong federal AGI and treated the Connecticut Social Security subtraction as an informal percentage rather than the traced $25,140 adjustment. It therefore used taxable income near $205,000 without correctly applying the $250 low-rate and $180 middle-rate recaptures that yield $11,917.18." +us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model invented a Connecticut deduction for federal income tax, federal itemized deductions, and an over-65 deduction that eliminate liability. Connecticut instead begins with federal AGI, applies the specified state modifications, and taxes $207,495.12 with no personal credit reduction." +us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model denied any Connecticut Social Security subtraction because income exceeds $75,000, but the applicable computation subtracts $25,140. It also used incorrect 3% and 4.5% bracket arithmetic and an unspecific recapture rather than the exact $250 and $180 recapture amounts." +us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model approximated Connecticut AGI and taxable income rather than deriving the traced $207,495.12 after the $25,140 state subtraction. Its bracket calculation also omitted the exact $250 low-rate and $180 middle-rate recaptures." +us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied an obsolete 2%–5.5% rate range and reduced taxable income to about $198,800. The applicable calculation taxes $207,495.12 under the 2026 Connecticut schedule and includes $430 of recapture." +us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model concluded that income above $100,000 eliminates the Connecticut Social Security adjustment, while the traced rule subtracts $25,140. That error left it taxing $234,414 instead of $207,495.12, and it also omitted the exact $250 and $180 recaptures." +us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model deducted Roth contributions and treated the Social Security adjustment inconsistently, producing an unsupported Connecticut AGI near $198,000. Only the traditional pretax amounts affect federal AGI, and the state computation then applies the specific $25,140 subtraction before taxing $207,495.12." +us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model reduced Connecticut AGI to $191,370 by over-exempting Social Security and then invented a $1,549.11 personal tax credit. The traced Connecticut AGI is $207,495.12, and the personal credit is zero at this income." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model used only an $8,904 Social Security subtraction and separately removed the state refund, leaving Connecticut AGI at $223,731. The state adjustments total $25,140 and produce Connecticut AGI of $207,495.12; the recaptures are specifically $250 and $180." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The answer supplied no computation supporting $7,800. Applying the $25,140 Connecticut subtraction to federal AGI and then the single-filer schedule plus $430 of recapture yields $11,917.18, so the estimate omitted a substantial portion of the taxable-income or rate calculation." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model treated high income as eliminating all Connecticut Social Security relief. The applicable adjustment subtracts $25,140, reducing Connecticut AGI to $207,495.12 before the schedule and the two recaptures are applied." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model began from an incorrect federal AGI of $225,235 and used a $26,711 Social Security exemption. The traced inputs instead give federal AGI of $232,635.12 and a $25,140 Connecticut subtraction, followed by $430 of recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used Connecticut taxable income of $212,320 instead of $207,495.12. The difference comes from failing to reproduce the exact federal-AGI starting point and $25,140 Connecticut subtraction, after which the two recaptures total $430." +us,scenario_120,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model removed all Social Security, invented a $15,000 Connecticut standard deduction, and applied a $200 property-tax credit. Connecticut uses a phased personal exemption rather than that standard deduction; here the personal credit does not reduce tax, and taxable income is $207,495.12 after the $25,140 state subtraction." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The unexplained $4,414 answer omits most of the Connecticut liability. Taxing the traced $207,495.12 under the single-filer schedule and adding the $250 and $180 recaptures produces $11,917.18." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked itemized deductions even though Connecticut liability here is computed from Connecticut AGI without those deductions. The applicable base is $207,495.12 after the $25,140 state subtraction, with no personal credit and $430 of recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model got close to the Connecticut AGI but used about $208,805 rather than $207,495.12 and estimated the recaptures as $500 and $180. The actual recaptures are $250 and $180, so both its taxable base and low-rate recapture were overstated." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model taxed approximately $235,031 of Connecticut income and therefore failed to apply the $25,140 Connecticut subtraction. The taxable base is $207,495.12, and the high-income additions consist of $250 low-rate recapture and $180 middle-rate recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model applied the schedule and recapture to about $234,000 instead of reducing federal AGI by the $25,140 Connecticut adjustment. Connecticut taxable income is $207,495.12, with only $430 added through the two recapture provisions." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model incorrectly said high income eliminates Connecticut Social Security relief and used roughly $234,414 as Connecticut AGI. The applicable Social Security adjustment contributes to a $25,140 subtraction, leaving $207,495.12 subject to the schedule and recaptures." +us,scenario_120,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The answer gave no usable derivation for $7,850. The correct state-modified taxable base is $207,495.12, and applying the Connecticut schedule plus $430 of recapture yields $11,917.18." +us,scenario_120,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model subtracted all federally taxable Social Security and the state refund, reducing Connecticut AGI to $197,049. Connecticut's traced subtraction is $25,140 in total, producing $207,495.12, and the liability also includes $430 of recapture." +us,scenario_120,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used the obsolete 3% and 5% lower Connecticut rates and an estimated taxable income of $201,651. The applicable 2026 lower rates are 2% and 4.5%, the taxable base is $207,495.12, and $430 of recapture must be added." +us,scenario_120,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model failed to apply the $25,140 Connecticut subtraction and instead taxed about $234,385. It then improperly reduced the requested liability with a $300 property-tax credit, while the traced personal-credit calculation reduces tax by zero." +us,scenario_120,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not return the requested state-income-tax output. No substantive Connecticut computation was submitted. +us,scenario_120,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model did not return the requested state-income-tax output. No substantive Connecticut computation was submitted. +us,scenario_120,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model reported zero without applying any Connecticut calculation. The household has $207,495.12 of Connecticut taxable income, which generates $11,917.18 after the schedule and $430 of recapture." +us,scenario_120,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model fully exempted Social Security, subtracted an already tax-exempt pension again, allowed a personal exemption at high income, and used obsolete 3% and 5% lower rates. The traced state subtraction is $25,140, the resulting taxable income is $207,495.12, and no personal credit reduces the tax." +us,scenario_120,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model used taxable income of $202,564 and applied a nonexistent $500 property-tax credit reduction. The traced taxable base is $207,495.12, the personal-credit calculation reduces liability by zero, and the two recaptures total $430." us,scenario_120,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_121,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"This model treated the listed disability benefits as SSDI and included $6,522 as taxable Social Security benefits in AGI, raising AGI from PolicyEngine's $25,665 to $32,187. That higher AGI created residual taxable income after medical itemized deductions, while PolicyEngine's AGI excludes those disability benefits and the deductions eliminate taxable income." -us,scenario_121,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"This model correctly excluded child support and disability benefits from taxable income but then defaulted to the standard deduction and ignored the large deductible medical expenses and health insurance premiums. PolicyEngine itemizes those medical deductions, which exceed the standard deduction and reduce taxable income to zero before nonrefundable credits." -us,scenario_121,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model computed childless EITC using $17,276 of earned income/AGI, which is the $25,665 wage amount reduced by $8,389 of employer-sponsored insurance premiums. Employer-sponsored insurance premiums do not reduce EITC earned income for this calculation; using the full $25,665 wage amount puts the single filer with no qualifying children above the childless EITC limit, yielding $0 refundable credits." -us,scenario_121,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model treated the head as eligible for childless EITC and phased the credit out from an earned income value of $17,276 rather than the listed annual wages of $25,665. That subtraction-based earned-income shortcut keeps the filer inside the childless EITC phaseout range; the correct EITC earned income exceeds the no-child limit, so the EITC is $0." -us,scenario_121,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model asserted childless EITC eligibility at $17,276 of earned income and AGI, matching a calculation that reduces wages by the employer-sponsored insurance premium. EITC earned income is not reduced that way here; the head's $25,665 in wages exceeds the single no-child EITC limit, leaving no refundable federal credit." -us,scenario_121,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a South Carolina disability Medicaid route based on disability status, medical expenses, and assets, but PolicyEngine assigns no Medicaid category for this 53-year-old. It also treated medical expenses and limited assets as satisfying eligibility even though the reference pathway never reaches a qualifying aged/disabled, medically needy, or MAGI category." -us,scenario_121,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly recognized that South Carolina is not an ACA expansion state and that SSI-related income would exceed an SSI-style limit, then overrode that result by assuming a medically needy or alternate disabled pathway. PolicyEngine does not place this head in an SSI-related, medically needy, or MAGI Medicaid category; medicaid_category is NONE." -us,scenario_121,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated being disabled and receiving disability benefits as qualification under South Carolina's Aged, Blind, and Disabled pathway. PolicyEngine does not assign the head to an ABD Medicaid category, and the trace shows no SSI receipt and no qualifying category despite the listed assets and disability status." -us,scenario_121,head_medicaid_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,The model applied a working-disabled Medicaid buy-in style test using a 250% FPL threshold and asset limit. PolicyEngine's South Carolina Medicaid computation does not qualify this person through that pathway; the head's category is NONE and MAGI at 1.61 FPL does not match an available category. -us,scenario_121,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model used disability status alone as a shortcut for Medicaid eligibility. PolicyEngine requires assignment to a specific South Carolina Medicaid category, and this 53-year-old disabled head is assigned medicaid_category = NONE." -us,scenario_121,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model reduced Medicaid eligibility to disabled plus low income. PolicyEngine finds no South Carolina Medicaid pathway for the head, and MAGI at 1.61 FPL does not qualify the person under any available category." -us,scenario_121,head_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model assumed South Carolina medically needy spend-down eligibility from high medical expenses. PolicyEngine does not assign a medically needy or other Medicaid category to this head, so medical expenses do not convert the NONE category into eligibility." -us,scenario_121,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not provide a usable value or explanation for head_medicaid_eligible. This is a missing-output contract failure rather than a substantive Medicaid-rule calculation. -us,scenario_121,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated meeting disability criteria as enough for South Carolina disability-based Medicaid. PolicyEngine requires an actual SSI-related or other qualifying Medicaid category, and the head has no SSI receipt and is assigned medicaid_category = NONE." -us,scenario_121,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted generic disability benefits into Social Security Disability Insurance and treated receipt as immediate Medicare qualification. The facts establish neither SSDI entitlement nor the required 24-month entitlement period, and the head is only 53." -us,scenario_121,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,The model assumed the listed disability benefits were SSDI. Generic disability status and benefits do not establish SSDI entitlement or completion of Medicare's 24-month disability waiting period. -us,scenario_121,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model explicitly invented completion of the 24-month SSDI waiting period from the instruction that facts remain constant during the year. That instruction does not supply prior SSDI entitlement, and generic disability benefits do not establish SSDI." -us,scenario_121,head_medicare_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model assumed both that the disability benefits were SSDI and that the 24-month waiting period had passed. Neither fact was listed, so both default to false under the prompt." -us,scenario_121,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,The model relabeled unspecified disability benefits as SSDI and then inferred satisfaction of the two-year Medicare waiting period. The prompt supplies neither SSDI entitlement nor the elapsed entitlement period. -us,scenario_121,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model asserted that the SSDI/Medicare waiting-period requirements were met despite no such fact being provided. At age 53, disability alone does not establish Medicare eligibility." -us,scenario_121,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,The model treated generic disability benefits as Social Security disability benefits that confer Medicare eligibility. It omitted the need for stated SSDI entitlement and the applicable 24-month entitlement period. -us,scenario_121,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model equated the listed disability benefits with SSDI and treated that unsupported classification as sufficient for Medicare. The head is under 65, and no qualifying SSDI entitlement duration or other Medicare pathway is listed." -us,scenario_121,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model treated disability status itself as a Medicare eligibility rule. For an individual under 65, the prompt must establish a qualifying disability-based entitlement or another special pathway; it does not." -us,scenario_121,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,The model concluded that disability plus unspecified disability benefits automatically activates Medicare's under-65 pathway. Those facts do not establish SSDI entitlement or completion of the required waiting period. -us,scenario_121,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,The model inferred that generic disability benefits were SSDI and that the statutory waiting period had elapsed. Both are unlisted facts that the prompt requires treating as false. +us,scenario_121,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model reclassified the generic disability-benefits input as SSDI and added $6,522 of purported taxable Social Security benefits to AGI, even though no Social Security income was listed. That inflated AGI and the medical-expense floor, producing $2,001 of taxable income instead of the zero taxable income obtained from the applicable itemized deduction." +us,scenario_121,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model used only the standard deduction and ignored the listed medical expenses and health-insurance premiums when comparing standard and itemized deductions. After the 7.5% AGI floor, the applicable itemized medical deduction eliminates taxable income, so applying 10% to $9,915 was the wrong computation." +us,scenario_121,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model set AGI equal to $17,276 by subtracting employer-sponsored insurance premiums from wages and omitted the $13,044 of disability benefits from the EITC phaseout income calculation. Including those benefits puts AGI above the childless EITC limit, reducing the credit to $0 rather than $139." +us,scenario_121,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model incorrectly used $17,276 as both earned income and AGI and therefore calculated a partially phased-out childless EITC. The disability benefits raise AGI beyond the applicable income ceiling, fully phasing the EITC out instead of leaving $127." +us,scenario_121,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model asserted childless EITC eligibility from $17,276 of earned income without applying the phaseout using AGI that includes the $13,044 of disability benefits. That higher AGI exceeds the childless EITC limit, so no $168 credit remains." +us,scenario_121,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated disability, medical expenses, and limited assets as jointly sufficient for South Carolina Medicaid without establishing a qualifying pathway. The head receives no SSI, has MAGI income of 1.61 times FPL, and is assigned no Medicaid category." +us,scenario_121,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly calculated that countable income exceeded the SSI-related limit, then reversed that result by asserting an unsupported disabled or medically needy pathway. Neither disability alone nor the listed medical expenses assign the head a Medicaid category, and the engine assigns NONE." +us,scenario_121,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model asserted that the wages and disability income fell within South Carolina's ABD income limit after disregards without performing a calculation that supports that assertion. Disability does not itself confer ABD eligibility, the head receives no SSI, and no qualifying Medicaid category is assigned." +us,scenario_121,head_medicaid_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model invented eligibility through a working-disabled Medicaid pathway using a 250% FPL test and an unexplained $30,320 income figure. The head's disability and assets do not place them in such a category under the applicable PolicyEngine rules; medicaid_category is NONE." +us,scenario_121,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the listed disability flag as automatic Medicaid eligibility. South Carolina requires a qualifying categorical pathway, and this 53-year-old head receives no SSI and is assigned medicaid_category NONE." +us,scenario_121,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model reduced Medicaid eligibility to being disabled with low income and never identified a South Carolina eligibility pathway. The head's MAGI is 1.61 times FPL, SSI receipt is zero, and no Medicaid category applies." +us,scenario_121,head_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model applied a medical-expense spend-down as though the listed expenses automatically created South Carolina medically needy eligibility. Those expenses do not assign a qualifying Medicaid category in this computation, so the head remains in category NONE." +us,scenario_121,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for head_medicaid_eligible, violating the required submission contract." +us,scenario_121,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model equated meeting the prompt's disability criterion with qualifying for SSI-related Medicaid. The head receives no SSI, and disability status alone does not satisfy an income-tested or categorical Medicaid pathway; the assigned category is NONE." +us,scenario_121,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted unspecified disability benefits into Social Security Disability Insurance and treated their receipt as sufficient for Medicare. The facts establish neither SSDI entitlement nor completion of the required waiting period, so the disabled 53-year-old does not qualify." +us,scenario_121,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model assumed that the disability benefits were SSDI and therefore triggered Medicare. The prompt supplies only generic disability benefits, while SSDI entitlement and the requisite waiting period are unlisted and therefore false." +us,scenario_121,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model explicitly assumed completion of Medicare's 24-month SSDI waiting period from the instruction that facts remain constant for the full year. Constancy during one tax year does not establish prior SSDI entitlement, and neither SSDI status nor a completed waiting period was listed." +us,scenario_121,head_medicare_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model labeled the unspecified disability benefits as SSDI and assumed the 24-month Medicare waiting period had passed. Both facts were unlisted and therefore false under the prompt, leaving the age-53 head without a Medicare eligibility pathway." +us,scenario_121,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"The model equated generic disability benefits with SSDI and applied the disability-based Medicare pathway. The record does not establish SSDI entitlement or two years of qualifying entitlement, so disability alone does not make this 53-year-old eligible." +us,scenario_121,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model invented satisfaction of the SSDI and Medicare waiting-period requirements. Those requirements are not stated and must be treated as false, so the head fails both the age-based and disability-entitlement pathways." +us,scenario_121,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model treated receipt of disability benefits as receipt of Social Security disability benefits and then treated that as sufficient for Medicare. The stated benefits are not identified as SSDI, and no qualifying entitlement duration is provided." +us,scenario_121,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model recast unspecified disability benefits as SSDI and concluded that disability itself conferred Medicare eligibility. Below age 65, qualifying SSDI entitlement and its required duration must be established; neither is present." +us,scenario_121,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model treated the disability flag alone as a Medicare qualification rule. Disability without a specified qualifying Medicare pathway, such as the required SSDI entitlement history, does not establish eligibility for a 53-year-old." +us,scenario_121,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model inferred that being disabled and receiving generic disability benefits automatically activated Medicare's under-65 disability pathway. That pathway requires qualifying entitlement conditions not listed in the household facts, so they are false under the prompt." +us,scenario_121,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,The model deemed the disability benefits 'consistent with SSDI' and inferred completion of the statutory waiting period. Consistency is not an input: SSDI entitlement and the completed waiting period were unlisted and therefore false. us,scenario_121,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_121,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model treated the $8,389 employer-sponsored insurance premium as a pre-tax employee salary reduction and subtracted it from FICA wages. The prompt supplies no Section 125 or pre-tax payroll-deduction status, so Social Security and Medicare apply to the full $25,665 of wages, yielding $1,963.37." -us,scenario_121,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages from $25,665 to $17,276 by assuming the employer-sponsored insurance premium was paid through a pre-tax salary reduction. Applying 7.65% to the full wage amount yields $1,963.37." -us,scenario_121,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model classified the $8,389 insurance premium as a pre-tax exclusion from Social Security and Medicare wages without any fact establishing that treatment. The FICA base is the full $25,665, producing $1,591.23 of Social Security tax and $372.14 of Medicare tax." -us,scenario_121,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model assumed that employer-sponsored health insurance premiums automatically reduce FICA-taxable wages. No pre-tax salary-reduction arrangement is listed, so the $8,389 premium is not deducted from the $25,665 payroll-tax base." -us,scenario_121,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model subtracted the $8,389 employer-sponsored insurance premium from wages by inventing pre-tax treatment that the household facts do not provide. Social Security and Medicare instead apply to all $25,665 of wages, totaling $1,963.37." -us,scenario_121,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model selected the full $25,665 wage base and the correct 6.2% and 1.45% rates, but then made arithmetic and transcription errors. The exact components are $1,591.23 and $372.14, totaling $1,963.37, while its submitted $1,911 does not follow from its own calculation." +us,scenario_121,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model treated the $8,389 employer-sponsored insurance premium as an employee pre-tax deduction from FICA wages even though the prompt does not identify it as an employee payroll contribution. The full $25,665 wage amount is subject to employee Social Security and Medicare taxes, yielding $1,963.37 rather than $1,322." +us,scenario_121,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $8,389 employer-sponsored insurance premium from gross wages to create a $17,276 FICA base. The payroll-tax base remains $25,665, producing $1,963.37 in employee Social Security and Medicare tax." +us,scenario_121,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model misclassified the listed $8,389 employer-sponsored insurance premium as a pre-tax employee contribution that reduces Social Security and Medicare wages. Applying the employee payroll-tax rates to the full $25,665 wage base produces $1,963.37." +us,scenario_121,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model reduced FICA-taxable wages by the employer-sponsored insurance premium without any prompt fact establishing an employee pre-tax payroll deduction. The correct base is the full $25,665 of gross wages, yielding $1,963.37." +us,scenario_121,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly deducted $8,389 of employer-sponsored insurance premiums from wages subject to FICA. Social Security and Medicare taxes apply to the full $25,665 wage amount here, totaling $1,963.37." +us,scenario_121,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's own calculation using the full wage base produced approximately $1,964.60, but it submitted $1,911 with no computation supporting that amount. The traced component calculation is $1,591.23 of Social Security tax plus $372.14 of Medicare tax, totaling $1,963.37, so the submitted value is a final arithmetic or transcription error rather than a rounding result." us,scenario_121,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_121,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model incorrectly waived the gross-income screen solely because the head is disabled. The household fails South Carolina's applicable gross-income limit before the $32,600 of claimed medical expenses or any other net-income deduction can affect eligibility." -us,scenario_121,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that a household containing a disabled person has no gross-income test. At $39,909 annually, the household fails the applicable South Carolina gross-income screen, so reducing net income to zero with medical, earned-income, and standard deductions does not establish eligibility." -us,scenario_121,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model placed the household on a net-income-only pathway based on disability and therefore applied large medical deductions. The applicable gross-income test is failed at $39,909, making the later near-maximum-allotment calculation irrelevant; it also supplied an allotment exceeding its own stated one-person maximum range." -us,scenario_121,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly identified that $39,909 exceeds the gross limit but then incorrectly treated disability as a categorical exemption from that test. Because the household fails the applicable gross-income screen, its detailed medical-deduction, asset, net-income, and allotment calculations never become operative." -us,scenario_121,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly replaced the gross-income screen with a net-income-only test for an elderly/disabled household. It then deducted duplicated premium inputs, inferred mortgage shelter costs from a balance despite the prompt setting unlisted payments to zero, and invented a partial offset, but eligibility already ends when $39,909 exceeds the gross limit." -us,scenario_121,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model incorrectly declared the gross-income test waived because the head is disabled. The household's $39,909 gross income fails the applicable South Carolina screen, so medical and shelter deductions cannot produce SNAP eligibility." -us,scenario_121,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model incorrectly treated disability as an automatic exemption from the gross-income test. The household fails that test at $39,909, so setting net income to zero through medical deductions and awarding the maximum allotment is the wrong eligibility sequence." -us,scenario_121,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly waived the gross-income limit based on the head's disability and proceeded directly to deductible medical expenses. Gross income of $39,909 fails the controlling eligibility screen, yielding no benefit regardless of the computed net income." -us,scenario_121,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model incorrectly exempted this household from the gross-income screen merely because the head is disabled. The earned-income, standard, and medical deductions it applied arise only after the household passes the controlling gross-income eligibility step, which $39,909 does not." -us,scenario_121,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model jumped directly to net-income deductions and a maximum allotment for a 'disabled SNAP household.' It omitted the applicable gross-income screen, which the household fails with $39,909 of annual income." -us,scenario_121,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model treated qualifying medical deductions as sufficient to reduce SNAP net income to zero without first applying the gross-income test. The household fails that prior screen at $39,909, so no maximum allotment is payable." -us,scenario_121,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model assumed that disability moved the household directly to an uncapped medical-deduction calculation. The applicable gross-income limit is exceeded by the household's $39,909 income, ending eligibility before net-income deductions or the maximum allotment are calculated." -us,scenario_121,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model incorrectly asserted that only the net-income test applies to this disabled head and therefore deducted medical expenses beyond gross income. The household instead fails the applicable gross-income screen at $39,909, so its estimated maximum benefit is unavailable." -us,scenario_121,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP output or explanation, violating the required structured-output contract." -us,scenario_121,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model incorrectly assumed that the head's disability activated a gross-test exemption and allowed medical deductions to reduce net income to zero. The household fails the applicable gross-income screen at $39,909, so the medical-deduction, resource-limit, and maximum-allotment calculations do not establish eligibility." +us,scenario_121,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the disabled status as waiving the SNAP gross-income test and then deducted medical expenses from net income. The household instead fails the gross-income screen on $39,909, so those deductions never produce SNAP eligibility." +us,scenario_121,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,The model asserted that an elderly or disabled household has no gross-income test and reduced net income to zero through the medical deduction. The applicable eligibility pathway rejects this household at the gross-income threshold before the benefit calculation. +us,scenario_121,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model assumed disability made the household subject only to the net-income test, then used medical deductions to infer a near-maximum benefit. The household's $39,909 gross income fails the controlling gross-income screen, yielding no entitlement." +us,scenario_121,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model correctly observed that gross income exceeds the ordinary limit but incorrectly converted the generic disabled status into an exemption from that test. Its later medical-deduction and resource calculations are immaterial because the household fails the gross-income eligibility step. +us,scenario_121,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly applied a disabled-household gross-test exemption and proceeded to deductions. It also inferred shelter costs from a mortgage balance despite the prompt specifying no mortgage payment, but the dispositive error is bypassing the gross-income limit that reduces SNAP to zero." +us,scenario_121,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated disability as automatically waiving the gross-income test. The household fails that test on approximately $3,326 of monthly gross income, so medical and shelter deductions do not establish eligibility." +us,scenario_121,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model incorrectly exempted the household from the gross-income screen solely because the head is disabled. The $39,909 annual income exceeds the applicable limit, preventing the net-income deductions and maximum allotment from being reached." +us,scenario_121,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model bypassed the gross-income test based on disabled status and used medical expenses to reduce net income to zero. The household instead fails the gross-income constraint, so the annual benefit is zero rather than the maximum allotment." +us,scenario_121,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model assumed the disabled head was exempt from the gross-income limit and moved directly to earned-income, standard, and medical deductions. The controlling gross-income test fails before those deductions can support eligibility." +us,scenario_121,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model jumped directly to net-income deductions for a disabled household and awarded the maximum allotment. It omitted the gross-income eligibility screen, which the household fails with $39,909 of annual income." +us,scenario_121,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model used qualifying medical costs to reduce SNAP net income to zero without first enforcing the gross-income limit. Because gross income exceeds that limit, no maximum-allotment calculation applies." +us,scenario_121,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model treated the medical deduction as sufficient to establish eligibility for a disabled household. The household is denied at the preceding gross-income screen, so its calculated $298 monthly allotment is inapplicable." +us,scenario_121,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model asserted that only the net-income test applies because the head is disabled. The applicable computation still enforces the gross-income threshold, which $39,909 exceeds, so deductions cannot generate eligibility." +us,scenario_121,snap,inkling,llm_error,categorical_eligibility,False,"The model reduced net income to zero using earned-income, standard, and medical deductions and awarded the maximum benefit. It omitted the controlling gross-income screen, under which the household is ineligible." +us,scenario_121,snap,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no SNAP value or explanation. This is a missing required output rather than a substantive benefit calculation. +us,scenario_121,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model treated disability as sufficient to bypass the gross-income test, then applied medical deductions and a resource test. The household fails the gross-income threshold on $39,909 before either calculation affects entitlement." us,scenario_121,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_121,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model chose the standard deduction and dismissed itemizing, even though the listed medical and health insurance expenses are large enough to eliminate the South Carolina taxable base after the medical-expense floor. It then invented a positive SC taxable income by subtracting only a standard deduction and personal exemption from wages and applying a flat 6.2% rate." -us,scenario_121,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer applies South Carolina deductions and brackets to a remaining positive taxable income instead of carrying through the itemized medical and health-premium deductions that reduce the taxable base to zero. The $686 estimate is consistent with a generic SC tax-bracket calculation that never accounts for the household's $30,000+ medical-expense deduction pathway." -us,scenario_121,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,The model subtracted only a projected single standard deduction from wages and taxed the remainder under SC brackets. It missed that the household itemizes because medical and health-premium expenses exceed the standard deduction and wipe out the South Carolina taxable base before rates apply. -us,scenario_121,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model included disability benefits in gross income and then used only a standard deduction plus vague exemptions or credits to reduce a positive SC tax. It missed both the exclusion of non-taxable support/disability treatment and, more importantly, the large itemized medical and health-premium deductions that eliminate South Carolina taxable income." -us,scenario_121,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated wages minus the standard deduction as South Carolina taxable income and then applied a graduated rate table to that amount. It ignored the household's large medical and insurance expenses, which make itemizing decisive and reduce the South Carolina taxable base to zero before any rate calculation." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s own calculation produced approximately $4,867 after applying the $26,200 combined deductions, but it submitted $6,552 without any computation supporting that figure. The submitted amount contradicts its stated taxable-income and rate calculation." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model taxed only 50% of Social Security instead of applying the provisional-income formula, which makes $27,200 taxable. It also invented disability-based standard-deduction amounts and treated charitable contributions as an AGI reduction despite claiming the standard deduction." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the separate $6,000 senior deduction and therefore used $50,446 instead of $43,446.18 of taxable income. That omission generated the excessive $5,736 liability." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model first derived about $45,646 of taxable income and about $5,011 of tax, then asserted that further unspecified deductions reduced the tax to $1,971. It neither calculated those deductions nor reconciled the submitted amount with its own rate calculation." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model reached approximately the correct deduction structure and taxable income, but its stated $2,336 ordinary-income tax is incompatible with applying the 10% and 12% brackets to roughly $41,722 of ordinary taxable income. It also taxed all qualified dividends at 15% after stating they fell in the 0% band." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed the TCJA rules expired for 2026, restored unreimbursed-employee-expense deductions and a personal exemption, and substituted post-sunset standard deductions and rate brackets. It consequently failed to use the applicable $20,200 standard deduction plus $6,000 senior deduction." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $1,824 of qualified dividends from AGI while still later treating them as part of taxable income. More importantly, it omitted the separate $6,000 senior deduction, leaving taxable income far above $43,446.18." -us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete itemized-deduction regime, deducting miscellaneous employee expenses and state income tax and restoring a personal exemption. It failed to take the applicable $20,200 standard deduction together with the $6,000 senior deduction." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model itemized deductions and claimed a personal exemption under an inapplicable post-TCJA-expiration framework. The correct deduction choice is the $20,200 age-and-blind standard deduction plus the separate $6,000 senior deduction, producing $43,446.18 of taxable income." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model reported mutually inconsistent figures: $71,468 of AGI minus a $29,600 standard deduction cannot produce $69,455 of taxable income, and itemized deductions cannot also be subtracted after choosing the standard deduction. The applicable computation uses $70,646.18 of AGI and total deductions of $27,200." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The answer’s $6,300 estimate reflects taxing substantially more than the $43,446.18 taxable-income base. It did not apply the full $20,200 standard deduction plus $6,000 senior deduction before calculating tax." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly restored the miscellaneous itemized deduction for employee expenses and a personal exemption, then used those obsolete items instead of the applicable standard and senior deductions. It also rounded the $1,519 capital loss to $1,500, misstating AGI." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Although the model correctly derived $70,646 of AGI, its $5,994 result reflects failure to subtract the full $27,200 consisting of the age-and-blind standard deduction and the separate senior deduction. The resulting taxable income should be $43,446.18." -us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted qualified dividends from provisional income and AGI, causing it to understate taxable Social Security and total AGI. It also omitted the separate $6,000 senior deduction and used estimated standard-deduction parameters instead of the applicable $20,200 amount." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly claimed the deductions fully offset income. After $27,200 of taxable Social Security is included, $70,646.18 of AGI minus the full $27,200 of deductions still leaves $43,446.18 taxable." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated tax-exempt income and listed itemized expenses as eliminating the federal liability, even though tax-exempt income does not create a deduction and the listed itemized deductions are smaller than the applicable deduction package. The correct computation leaves $43,446.18 of taxable income rather than zero." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed a 2026 TCJA sunset, restored a personal exemption, and deducted unreimbursed employee expenses through itemization. It failed to apply the governing $20,200 standard deduction plus $6,000 senior deduction." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model added a $1,000 nonitemizer charitable deduction that is not part of the traced deduction calculation and used an imprecise standard-deduction amount. The deductions total exactly $27,200, producing $43,446.18 of taxable income and $4,746.66 of tax." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the relevant age, blindness, and senior deductions but did not calculate their exact amounts or apply the exact 2026 brackets. The precise taxable-income base is $43,446.18, and the rate calculation on that base yields $4,746.66 rather than $4,957." -us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated absence of earned income as eliminating income tax, even though pensions, IRA distributions, dividends, and taxable Social Security are included in gross income. After deductions, these sources leave $43,446.18 of taxable income, well above the filing threshold." -us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly used a post-TCJA-sunset regime with deductible employee expenses, state income tax, and a restored personal exemption. It should instead subtract the $20,200 standard deduction and $6,000 senior deduction from AGI." -us,scenario_122,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used obsolete post-sunset itemization and rate assumptions, including deductions for employee expenses and an estimated state income tax. It also reported $50,436 of taxable income even though $70,646 minus its claimed $20,210 of itemized deductions equals $50,436 only before accounting for the deduction regime actually in force." -us,scenario_122,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. Its response therefore failed the required output contract. -us,scenario_122,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model acknowledged positive taxable income but then asserted that senior deductions, personal exemptions, and credits fully eliminated it without computing any such offset. The applicable deductions leave $43,446.18 taxable, and no nonrefundable credit reduces the resulting tax to zero." -us,scenario_122,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model’s own detailed calculation ended at $5,544, but it submitted $9,505 without any supporting step. It also wrongly treated the negative $1,447 long-term capital gain as positive preferential-rate income and omitted the separate $6,000 senior deduction." +us,scenario_121,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model explicitly dismissed itemizing as less favorable despite $30,000 of medical expenses plus health-insurance premiums; after the medical-expense AGI floor, those deductions exceed the standard deduction and eliminate South Carolina taxable income. It also invented SSDI treatment for the generic disability-benefit input and applied an erroneous flat-rate calculation, but the omitted medical itemization alone separates its $400 answer from zero." +us,scenario_121,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The $686 estimate implies that the model retained positive South Carolina taxable income after ordinary state deductions. It failed to apply the household’s itemized medical deduction from $30,000 of medical expenses and qualifying premiums, which reduces taxable income to zero." +us,scenario_121,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model subtracted only a projected standard deduction from wages and taxed the resulting $10,265. It failed to choose the larger itemized deduction generated by the household’s $30,000 of medical expenses and qualifying premiums, which eliminates South Carolina taxable income after the medical-expense floor." +us,scenario_121,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated nearly all wages and disability benefits as gross taxable income, subtracted only a standard deduction, and then applied unsupported estimated exemptions or credits. It omitted the much larger itemized medical deduction from $30,000 of medical expenses and qualifying premiums, which reduces South Carolina taxable income to zero." +us,scenario_121,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used wages minus an estimated standard deduction as South Carolina taxable income and ignored the household’s itemized medical expenses. The deductible amount remaining after the AGI floor, including qualifying premiums, exceeds that standard deduction and eliminates taxable income; its stated graduated-rate arithmetic also does not produce the submitted $94.26." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's written calculation reaches approximately $4,867 but submits $6,552, a value unsupported by its own derivation. It also raises taxable income by $1,000 to $44,446 instead of subtracting the full $27,200 of applicable deductions from $70,646 to reach $43,446." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model taxed only 50% of Social Security instead of applying the statutory provisional-income formula, which makes $27,200 taxable. It invented an additional deduction for disability, treated charitable contributions as an AGI reduction while also taking the standard deduction, and then submitted $1,065 despite deriving $3,668." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the separate $6,000 senior deduction and therefore used $50,446 rather than $43,446 of taxable income. That omission also pushed part of the qualified dividends into the 15% band instead of leaving all $1,824 in the 0% band." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model first derived taxable income near $45,646 and tax near $5,011, then asserted without calculation that further deductions reduced the tax to $1,971. The applicable deductions total $27,200, producing $43,446 of taxable income and $4,746.66 of tax." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model approximated the standard deduction and then miscomputed the ordinary-bracket tax on roughly $41,722 as only $2,336. With exact taxable income of $43,446 and $1,824 of qualified dividends in the 0% band, the remaining ordinary income produces $4,746.66 of tax." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed the TCJA rules expired for 2026, restoring a personal exemption, pre-TCJA brackets, and the miscellaneous itemized deduction for employee expenses. Current 2026 law instead supplies a $20,200 standard deduction plus the $6,000 senior deduction, leaving $43,446 taxable." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $1,824 of qualified dividends from provisional income and AGI, producing $68,822 instead of $70,646. It also omitted the separate $6,000 senior deduction, so it used $48,622 rather than $43,446 of taxable income." +us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applied a restored personal exemption, a miscellaneous employee-expense deduction, an inferred state-income-tax itemized deduction, and pre-TCJA rates. The correct pathway takes the $20,200 standard deduction and $6,000 senior deduction, with no personal exemption, producing $43,446 taxable income." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used post-sunset itemization and a restored personal exemption instead of the 2026 standard deduction and separate senior deduction. Those current-law deductions total $27,200 and reduce taxable income to $43,446, not $47,320." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model reported mutually inconsistent figures: $69,455 cannot equal $71,468 minus a $29,600 standard deduction, and itemized deductions cannot also be subtracted after choosing the standard deduction. The correct AGI is $70,646 and the applicable deductions total $27,200, leaving $43,446 taxable." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The answer reflects a rough liability estimate without applying the exact $20,200 standard deduction and $6,000 senior deduction. Those deductions reduce $70,646 of AGI to $43,446, after which the qualified-dividend worksheet yields $4,746.66." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly restored the miscellaneous itemized deduction and personal exemption and used itemized deductions instead of current-law standard and senior deductions. It also rounded the $1,519 capital loss to $1,500, but the controlling error is its $48,492 taxable income rather than $43,446." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Although it found the correct $70,646 AGI, the submitted tax implies that it omitted or understated the separate $6,000 senior deduction. The full $27,200 deduction produces $43,446 taxable income and keeps all qualified dividends in the 0% band." +us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted qualified dividends from provisional income and AGI, causing taxable Social Security to be understated as $26,289.75 rather than the $27,200 maximum. It also omitted the $6,000 senior deduction and used an estimated $19,570 standard deduction rather than the combined $27,200 deductions." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that age and blindness deductions eliminate taxable income. After the $20,200 standard deduction and $6,000 senior deduction, $43,446 remains taxable and produces $4,746.66 of tax." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated tax-exempt income and listed itemized expenses as though they eliminated the federal liability. The taxpayer instead uses the $20,200 standard deduction plus $6,000 senior deduction, leaving $43,446 taxable, and no listed nonrefundable credit reduces the resulting tax to zero." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied a TCJA sunset, restored itemized employee-expense deductions and a personal exemption, and used pre-TCJA ordinary rates. Current 2026 law instead yields $43,446 taxable income after the $20,200 standard deduction and $6,000 senior deduction." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model added a $1,000 nonitemizer charitable deduction that is not part of the traced deduction calculation. The correct deductions are exactly $20,200 plus $6,000, producing $43,446 taxable income; the added charitable deduction explains its near-reference understatement." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the relevant deduction categories but did not apply their exact 2026 amounts or the exact bracket calculation. Using $43,446 of taxable income and the qualified-dividend worksheet yields $4,746.66 rather than $4,957." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model wrongly treated absence of earned income as eliminating income tax and claimed income fell below the filing threshold. Pension, IRA, taxable Social Security, dividends, and interest leave $43,446 taxable after deductions, so unearned income generates $4,746.66 of federal tax." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model assumed a TCJA sunset, deducted miscellaneous employee expenses and inferred Minnesota income tax, restored a personal exemption, and applied pre-TCJA rates. The correct 2026 deduction pathway uses the $20,200 standard deduction plus $6,000 senior deduction and leaves $43,446 taxable." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used post-TCJA-reversion itemization, including miscellaneous employee expenses and an inferred state-income-tax deduction, while its stated $50,436 taxable income does not follow from $70,646 minus $20,210. Current law instead deducts $27,200 and applies the 10% and 12% brackets to the ordinary-income portion of $43,446." +us,scenario_122,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model restored a personal exemption and miscellaneous itemized deduction and inferred an unlisted Minnesota income-tax deduction. The taxpayer instead takes the $20,200 standard deduction and $6,000 senior deduction, producing $43,446 taxable income under current 2026 rates." +us,scenario_122,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response failed the submission contract." +us,scenario_122,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model acknowledged positive taxable income but then invented personal exemptions or credits sufficient to eliminate it. No such offset applies: the standard and senior deductions leave $43,446 taxable and no nonrefundable credit reduces the $4,746.66 tax." +us,scenario_122,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's detailed computation arrives at $5,544 but submits $9,505, so the final value does not follow from its own reasoning. It also omitted the $6,000 senior deduction and incorrectly treated the $1,447 capital loss as preferential long-term capital-gain income after already deducting the net $1,519 loss." +us,scenario_122,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model excluded all Social Security from AGI even though provisional income makes $27,200 taxable, invented a $1,875 elderly/disabled credit despite the income limits, and treated a charitable deduction as a credit. Its submitted $5,065 also exceeds the $3,933.60 tentative tax it claimed after purported reductions, making the final arithmetic internally inconsistent." us,scenario_122,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_122,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model converted aged, blind, and disabled characteristics into categorical Medicaid eligibility without applying Minnesota's actual pathway gates. It also asserted away asset and income limits by invoking MAGI-style no-asset-test logic for elderly/disabled pathways, while the engine found no qualifying category and no Medicaid eligibility." -us,scenario_122,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model correctly recognized that MAGI adult Medicaid fails because income exceeds the 138% FPL limit, but then overrode that failure with an unsupported elderly/disabled alternative pathway. PolicyEngine's category determination is NONE, so age 73, blindness, and disability do not themselves establish Medicaid eligibility." -us,scenario_122,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly recognized that no wage or salary income was provided, then overrode that rule by inventing $1,270.64 of employee payroll tax on non-wage taxable income. Payroll tax is not computed from taxable retirement, Social Security, dividend, interest, or capital-gain income; with unlisted wages treated as zero, the payroll-tax base is zero." -us,scenario_122,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable payroll_tax value or explanation. The required computation produces zero because the household has no listed wages or self-employment income, and payroll tax does not apply to the listed retirement and investment income." +us,scenario_122,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated being age 73, blind, and disabled as automatic categorical eligibility and conflated Medicare Savings Programs such as QMB and SLMB with full Medicaid eligibility. It also dismissed resource limits despite the disclosed $44,000 in bank assets and $177,000 in stock assets; the head qualified through none of Minnesota’s Medicaid pathways." +us,scenario_122,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model correctly found income far above the 138% FPL MAGI limit, then reversed that result solely because the head was elderly, blind, and disabled. Those characteristics do not automatically confer Medicaid eligibility, and the head qualified through no alternative category." +us,scenario_122,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that no wage earnings were provided and that payroll tax therefore equals zero, then contradicted that reasoning by imposing an unsupported $1,270.64 tax on unspecified ""taxable income."" Retirement and investment income is not an employee Social Security or Medicare tax base." +us,scenario_122,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable payroll_tax output or explanation, violating the required structured-output contract." us,scenario_122,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model treated the Minnesota Social Security subtraction as mostly phased out and constructed taxable income near $52,000. It missed the $27,200 Social Security subtraction and $4,977.18 charitable subtraction that reduce taxable income to $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that age and blindness deductions eliminate Minnesota taxable income. The applicable subtractions leave $19,219 taxable, not zero, yielding $1,028.22 at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model correctly approached the Social Security and charitable subtractions, then invented an additional senior subtraction or elderly credit that erased the remaining liability. Minnesota taxable income remains $19,219 after the traced subtractions, with no identified nonrefundable credit reducing the $1,028.22 basic tax to zero." -us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model overstated deductions and unidentified elderly or disability credits until it drove the tax to zero. The actual Minnesota subtractions total $32,177.18 and leave $19,219 taxable income." -us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an estimated taxable base near $44,000 because it understated the combined Social Security and charitable subtractions. Those subtractions reduce Minnesota taxable income to $19,219, all within the 5.35% bracket." -us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented a $15,000 Minnesota pension/IRA subtraction and rebuilt Minnesota deductions from federal taxable income. The trace instead applies $32,177.18 of Minnesota subtractions, principally $27,200 for Social Security and $4,977.18 for charity, leaving $19,219 taxable." -us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted or overstated age, blindness, disability, itemized, and Social Security relief and therefore erased the tax base. The applicable Minnesota subtractions leave $19,219 subject to the 5.35% rate." -us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted only itemized deductions and a personal exemption from AGI, producing $48,529 of taxable income. It omitted the traced $27,200 Social Security subtraction and $4,977.18 charitable subtraction that produce $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model relied on a standard deduction and age/blind additions while omitting the Minnesota subtractions totaling $32,177.18. This inflated taxable income from $19,219 to $51,946." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $2,420 reflects a generic AGI-minus-standard-deduction estimate rather than the Minnesota subtraction calculation. Applying the $27,200 Social Security and $4,977.18 charitable subtractions yields $19,219 taxable income and $1,028.22 of tax." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied only a partial Social Security subtraction and consequently retained too much taxable income. The full traced $27,200 Social Security subtraction plus $4,977.18 charitable subtraction reduces taxable income to $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model retained $51,665 of taxable income because it understated Minnesota's Social Security and charitable subtractions. The correct subtraction total is $32,177.18, leaving $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted Social Security and an estimated standard deduction but stopped at $25,176. It failed to apply the traced deduction structure, including the $4,977.18 charitable subtraction, that leaves $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used nonexistent 2.85% and 4.80% Minnesota income-tax brackets instead of the 5.35% first bracket. It also left taxable income at $28,520 rather than applying the traced subtractions to reach $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated senior and disability relief as sufficient to eliminate taxable income. Minnesota's traced subtractions leave $19,219 taxable and therefore $1,028.22 of tax." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and nonrefundable credits reduce liability to a nonpositive amount without computing them. The actual subtraction calculation leaves $19,219 taxable, and the 5.35% basic tax is positive." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model was close but used an estimated taxable base rather than the traced $19,219 amount. Applying 5.35% to the exact taxable income gives $1,028.22, not $1,094." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model claimed to apply the Social Security subtraction but still estimated $49,900 of taxable income, which fails to reflect that subtraction. Minnesota's total $32,177.18 of subtractions leaves only $19,219 taxable." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model approximated taxable income as $21,055 instead of calculating the exact Minnesota subtraction amounts. The traced $27,200 Social Security and $4,977.18 charitable subtractions leave $19,219, lowering tax to $1,028.22." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $2,640 answer implies that it retained far more pension, IRA, investment, or Social Security income in the taxable base. The Minnesota subtractions reduce the $70,646.18 AGI to $19,219 of taxable income." -us,scenario_122,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced Minnesota taxable income to zero through unspecified age-based exemptions and credits. The traced subtractions leave $19,219 taxable, producing positive tax." -us,scenario_122,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $19,600 age-and-blindness standard deduction and stopped at $23,844 taxable income. The actual Minnesota subtraction calculation includes the $4,977.18 charitable subtraction and yields $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model constructed $53,623 of taxable income from itemized deductions and omitted Minnesota's $27,200 Social Security subtraction and $4,977.18 charitable subtraction. Those traced subtractions reduce taxable income to $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric value or explanation for the requested output, so its response could not be parsed or evaluated as a substantive estimate." -us,scenario_122,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model reached $18,690 by applying estimated standard and personal-exemption amounts after the Social Security subtraction. Minnesota's traced subtraction calculation instead yields $19,219 taxable income, whose 5.35% tax is $1,028.22." -us,scenario_122,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented a senior age-65 subtraction large enough to eliminate taxable income. The applicable Minnesota subtractions leave $19,219 taxable rather than zero." -us,scenario_122,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted Minnesota's $27,200 Social Security and $4,977.18 charitable subtractions, then improperly treated a property-tax refund estimate as a nonrefundable income-tax reduction. Those subtractions yield $19,219 taxable income, and the separate property-tax refund does not reduce this output." -us,scenario_122,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model correctly ruled out the Working Family Credit and Child Tax Credit but then incorrectly awarded a renter's property tax refund. Its own household-income estimate of about $77,500 does not produce an eligible 2026 Minnesota renter credit, so applying a deemed-property-tax percentage, copayment, and estimated phaseout generated a nonexistent $2,265 credit." -us,scenario_122,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model asserted renter-credit qualification from rent and household income without applying the 2026 Minnesota eligibility limit or schedule. The household receives no renter credit and has no other refundable Minnesota credit, so the $2,000 estimate comes entirely from an inapplicable eligibility pathway." -us,scenario_122,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"The model applied a senior/disabled renter-credit schedule and high-income copayment after assuming the household qualified. The applicable 2026 rules yield no renter credit at this household's income, so the deemed-property-tax calculation cannot create the submitted $1,070 refundable credit." -us,scenario_122,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model incorrectly treated age or disability as providing an eligible Minnesota renter-credit pathway at this household income. Applying the rent percentage, household-income adjustment, and schedule after that mistaken eligibility decision produced the unsupported $2,228 amount." -us,scenario_122,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, violating the required structured-output contract. It therefore never reported the zero produced by the Minnesota refundable-credit calculation." -us,scenario_123,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable child1_chip_eligible output. The required result is 0 because child 1 is Medicaid-ineligible and fails Pennsylvania CHIP’s age or income criteria. -us,scenario_123,child1_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated Pennsylvania’s general description of CHIP coverage through age 18 and availability of paid CHIP at higher incomes as sufficient for eligibility. It failed to apply the program-specific enrollment cutoff that excludes this 16-year-old and the applicable CHIP income threshold, which the family’s approximately $145,002 income exceeds." +us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model treated the Minnesota Social Security subtraction as negligible and substituted an itemized-deduction calculation. Minnesota instead subtracts $27,200 of Social Security and $4,977.18 for charitable contributions, reducing taxable income to $19,219 rather than about $52,000." +us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model concluded that age, blindness, the standard deduction, and Social Security exclusion eliminated all taxable income. The applicable Minnesota subtractions reduce federal AGI to $19,219, not zero, leaving $1,028.22 of basic tax." +us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model correctly approached the $27,200 Social Security subtraction and charitable subtraction, then invented an additional senior subtraction or elderly/disabled credit that erased the remaining liability. Minnesota taxable income remains $19,219, and no identified nonrefundable adjustment reduces its $1,028.22 basic tax to zero." +us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model subtracted the full $32,000 gross Social Security benefit instead of the $27,200 amount included in AGI and then treated age/blind deductions and unidentified credits as sufficient to eliminate tax. The traced Minnesota subtractions total $32,177.18 and leave $19,219 taxable." +us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model left taxable income near $44,000 by understating the Minnesota Social Security and charitable-contribution subtractions. Those subtractions total $32,177.18 and reduce $70,646.18 of AGI to $19,219." +us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented a $15,000 Minnesota pension/IRA subtraction and rebuilt taxable income using unsupported itemized deductions, including unreimbursed employee expenses. The applicable adjustments are principally the $27,200 Social Security subtraction and $4,977.18 charitable subtraction, leaving $19,219 taxable rather than $10,010." +us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted deductions and invoked an elderly/disability credit to drive taxable income to zero. Minnesota's traced subtractions leave $19,219 of taxable income, so the basic tax remains positive." +us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted only itemized deductions and a personal exemption, leaving $48,529 taxable. It omitted the Minnesota subtractions totaling $32,177.18, including $27,200 of Social Security and $4,977.18 of charitable contributions." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied only estimated standard and age/blind deductions to AGI and left $51,946 taxable. It failed to apply the traced Minnesota Social Security and charitable-contribution subtractions that produce $19,219 of taxable income." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer reflects a generic AGI-minus-standard-deduction estimate and does not account for the specific $32,177.18 of Minnesota subtractions. Those subtractions produce $19,219 of taxable income and $1,028.22 of tax." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied only a partial Social Security subtraction and left too much income subject to Minnesota tax. The Social Security subtraction is $27,200, and the additional $4,977.18 charitable subtraction brings taxable income to $19,219." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model left $51,665 taxable by materially understating the Minnesota Social Security subtraction and omitting the traced charitable subtraction. The correct subtraction total is $32,177.18, leaving $19,219 taxable." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model applied the $27,200 Social Security subtraction but substituted an estimated standard deduction and stopped at $25,176. It missed the applicable charitable-contribution subtraction within Minnesota's traced $32,177.18 total, which leaves $19,219 taxable." +us,scenario_122,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a nonexistent Minnesota rate schedule of 2.85% and 4.80% instead of the applicable basic tax schedule. It also used $28,520 of taxable income rather than the traced $19,219 because it did not apply the $4,977.18 charitable subtraction correctly." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that standard and senior/disabled deductions eliminated taxable income. The applicable Minnesota subtractions leave $19,219 taxable, which generates $1,028.22 before refundable credits." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The zero answer implies that the model treated deductions or nonrefundable credits as exceeding the basic tax without identifying any such valid adjustment. The traced calculation leaves $19,219 taxable and a positive $1,028.22 liability." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model recognized the Social Security and charitable subtractions but estimated their interaction with the standard deduction rather than using the resulting Minnesota taxable income of $19,219. That taxable base produces $1,028.22, not $1,094." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model claimed to apply the Social Security subtraction but still left about $49,900 taxable, which is inconsistent with subtracting $27,200 from $70,646.18. It also omitted the $4,977.18 charitable subtraction; the resulting taxable income is $19,219." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model estimated taxable income at $21,055 instead of applying the traced Minnesota subtraction total exactly. The $27,200 Social Security subtraction and $4,977.18 charitable subtraction reduce AGI to $19,219, lowering the basic tax to $1,028.22." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The answer implies that substantial pension, IRA, investment, and Social Security income remained taxable after only a rough subtraction estimate. Minnesota subtractions total $32,177.18 and leave only $19,219 subject to its basic tax calculation." +us,scenario_122,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated age-based exemptions and other subtractions as eliminating Minnesota taxable income. They leave $19,219 taxable and therefore do not reduce the liability to zero." +us,scenario_122,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $19,600 standard deduction after subtracting Social Security and arrived at $23,844 taxable. It failed to reproduce the applicable $4,977.18 charitable subtraction and the traced $19,219 taxable-income result." +us,scenario_122,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model deducted generic itemized expenses from AGI but omitted the $27,200 Minnesota Social Security subtraction. It also failed to use the $4,977.18 charitable subtraction, leaving $53,623 taxable instead of $19,219." +us,scenario_122,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model's approximate $50,000 taxable base omits most of the $32,177.18 in Minnesota subtractions. Applying the $27,200 Social Security and $4,977.18 charitable subtractions leaves $19,219." +us,scenario_122,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not satisfy the submission contract." +us,scenario_122,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model added a personal exemption and used estimated standard-deduction components, producing $18,690 taxable. The traced Minnesota calculation instead applies $32,177.18 of subtractions to AGI and yields $19,219 taxable." +us,scenario_122,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the capital loss, senior treatment, Social Security subtraction, and standard deduction as sufficient to reduce taxable income to zero. Minnesota's actual subtraction sequence leaves $19,219 taxable and $1,028.22 of tax." +us,scenario_122,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $27,200 Minnesota Social Security subtraction and the $4,977.18 charitable subtraction, then improperly used a property-tax-refund estimate to reduce this income-tax output. Property tax relief is not a nonrefundable adjustment to the requested liability, and Minnesota taxable income is $19,219." +us,scenario_122,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model began from an incorrect $52,092 federal AGI and applied an unsupported $30,100 Minnesota standard deduction. Federal AGI is $70,646.18, and the traced $32,177.18 of Minnesota subtractions produces $19,219 taxable income." +us,scenario_122,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model recognized that household income was about $77,500 but still invented a positive renter credit by subtracting an estimated threshold amount and then adding an unsupported adjustment to reach $2,265. At that household income, the 2026 Minnesota renter-credit schedule yields zero, and the working-family and child credits are also zero." +us,scenario_122,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model treated payment of rent as sufficient for a Minnesota renter credit and assigned an estimated maximum-like amount without applying the income schedule. Approximately $77,460 of household income phases the renter credit to zero." +us,scenario_122,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model applied a supposed senior/disabled renter-credit schedule that left a $1,070 refund after a high-income copayment. Minnesota’s 2026 income phaseout yields zero at this household income, and senior or disabled status does not preserve a positive credit." +us,scenario_122,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model converted 17% of rent into deemed property tax and then estimated a positive credit without completing the applicable income phaseout. The 2026 renter-credit schedule returns zero at approximately $77,460 of household income, including for a senior or disabled claimant." +us,scenario_122,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for state_refundable_credits, violating the required submission contract." +us,scenario_123,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable child1_chip_eligible value or explanation, violating the required output contract." +us,scenario_123,child1_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated age 16 as satisfying Pennsylvania CHIP’s age rule and assumed paid CHIP extends eligibility regardless of income. Under the traced Pennsylvania program structure, the child fails the enrollment age cutoff, and household income of approximately $145,002 also exceeds the CHIP threshold." +us,scenario_123,child1_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model declared eligibility solely from the child’s age without applying Pennsylvania’s specific CHIP enrollment cutoff or the income test. The child fails the program’s age criterion, and the approximately $145,002 household income independently exceeds the CHIP threshold." us,scenario_123,child1_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s submitted $5,860 contradicts its own calculations and final narrative, which successively state $5,440 and $8,660. It also omitted the $2,000 non-itemizer charitable deduction; the joint return uses $34,200 of deductions and yields $5,200.24 after the $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the outstanding mortgage balance, used an obsolete standard deduction, and counted two children when only one exists. The joint return instead takes the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction and claims one $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used an estimated $31,500 standard deduction and a $2,000 CTC, then replaced its computed result with an unsupported lower number. The applicable deductions total $34,200 and the nonrefundable CTC is $2,200, producing $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model incorrectly denied the CTC because the child earned $45,000 and invented mortgage interest from the loan balance. A child’s wages do not automatically fail the qualifying-child support test; the parents claim the full $2,200 CTC and use $34,200 of deductions." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model identified the correct AGI, standard deduction, and $2,200 CTC but approximated the 2026 brackets instead of applying their exact thresholds. It also omitted the $2,000 non-itemizer charitable deduction, so taxable income is $65,802 and the result is $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model eventually used extended-law concepts but retained estimated 2026 deduction and bracket values and a $2,000 CTC. The exact computation uses $34,200 of deductions, the 2026 brackets, and a $2,200 CTC, yielding $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The submitted $15,300 does not follow the model’s own narrative, which computes approximately $7,300. That narrative also wrongly subtracts itemized deductions despite saying they are below the standard deduction and omits the $2,000 non-itemizer charitable deduction and $200 of CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model applied a pre-TCJA sunset regime and added the dependent child’s separate-return tax to the parents’ requested tax-unit output. The output covers the joint tax unit only and uses the enacted 2026 standard deduction, non-itemizer charitable deduction, rates, and $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model added the child’s separate income tax to the requested joint tax-unit value and applied an obsolete post-sunset regime. It also invented mortgage interest and state taxes; the parents’ joint calculation alone yields $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 of wages in the parents’ joint AGI. The joint tax unit’s AGI is $100,002, and after $34,200 of deductions and the $2,200 CTC its tax is $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model both described $145,002 as joint AGI and added a separate tax on the same child’s wages, double counting that income across filing units. The requested value includes only the parents’ joint return with $100,002 of AGI." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model applied obsolete personal exemptions, rates, and a $1,000 CTC, then added the child’s separate-return tax. The requested joint tax-unit output excludes that separate tax and follows the enacted 2026 rules, yielding $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model treated all $145,002 as joint tax-unit AGI and incorrectly phased out the CTC at that income. The parents’ AGI is $100,002, and they receive the full $2,200 CTC for the qualifying 16-year-old." -us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added the child’s separately computed $3,312.08 tax to a variable belonging to the parents’ joint tax unit. Its parents-only calculation also used approximate deductions and brackets and omitted the $2,000 non-itemizer charitable deduction." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 wages in married-joint income. Those wages belong to the child’s separate tax return, leaving the parents with $100,002 of AGI and $5,200.24 of tax after the nonrefundable CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The answer is consistent with taxing the household’s combined $145,002 rather than the parents’ $100,002 joint-tax-unit AGI. The requested output excludes the child’s separate earned income and separate tax liability." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model explicitly included the child’s $45,000 wages in joint AGI. The parents’ joint AGI is $100,002; applying $34,200 of deductions and the $2,200 CTC yields $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model correctly recognized the $2,000 non-itemizer charitable deduction but wrongly included the child’s $45,000 wages in joint AGI. Removing those separately reported wages reduces taxable income to $65,802 and tax after the CTC to $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model’s stated regular tax corresponds to including the child’s wages in the parents’ tax base and it used a $30,000 standard deduction. The parents instead have $100,002 of AGI and $34,200 of total deductions." -us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model combined the head’s and child’s wages in the parents’ income and also invoked unlisted mortgage interest. The child’s wages are outside the joint return, and only supplied deductible amounts enter the deduction comparison." -us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model applied an obsolete sunset regime with personal exemptions and a $1,000 CTC, then added the child’s separate-return tax. The requested output is the parents’ joint tax-unit amount under enacted 2026 rules and excludes the child’s separate liability." -us,scenario_123,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model used obsolete personal exemptions, 10%/15% brackets, and a $1,000 CTC, and added the child’s separate tax. The parents’ joint tax unit instead uses $34,200 of deductions, current 2026 brackets, and a $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $2,000 charitable deduction allowed in addition to the $32,200 standard deduction, leaving taxable income $2,000 too high. It also discussed adding the child's separate tax despite the requested variable belonging to the parents' joint tax unit, and its submitted $5,860 contradicts its own calculations." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the outstanding balance, used an obsolete $29,200 standard deduction, and fabricated a second child and a $4,000 CTC. The correct joint-return computation uses no unlisted mortgage interest, deducts $32,200 plus the $2,000 non-itemizer charitable deduction, and claims one $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used an estimated $31,500 standard deduction and omitted the separate $2,000 non-itemizer charitable deduction. Those errors prevented it from reaching $65,802 of taxable income before applying the 2026 brackets and $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented $29,400 of mortgage interest from the mortgage balance and incorrectly denied the CTC because the child earned $45,000. A qualifying child is not disqualified by an earned-income ceiling; the support test controls, and the joint return receives the full $2,200 credit after the prescribed deductions." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model identified the correct AGI, $32,200 standard deduction, and $2,200 CTC but omitted the additional $2,000 non-itemizer charitable deduction and used approximate bracket arithmetic. Taxable income is $65,802 and the resulting pre-credit tax is $7,400.24, not roughly $7,610." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"After abandoning its pre-TCJA calculation, the model still used an estimated $30,750 standard deduction and omitted the $2,000 charitable deduction for non-itemizers. The applicable deductions total $34,200, and the applicable CTC is $2,200 rather than $2,000." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model stated that $18,557 of itemized deductions exceeded a roughly $32,200 standard deduction, reversed the comparison, and then submitted $15,300 despite reasoning to about $7,300. The joint return instead takes the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction." +us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model applied an obsolete pre-TCJA regime with personal exemptions, 15% brackets, and a $1,000 CTC, then added the child's separate-return tax to the requested joint-tax-unit output. The correct output contains only the parents' joint return under the applicable 2026 deductions, rates, and $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model used a pre-TCJA tax regime, invented mortgage interest and state taxes, and added the child's separate-return liability. The requested tax-unit value excludes the child's wages and tax and uses $34,200 of joint-return deductions followed by a $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model included the child's $45,000 wages in the parents' joint AGI, producing $145,002 instead of $100,002. The child's earned income is not reported on the parents' joint return, so the applicable taxable income is $65,802." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model both described combined AGI as $145,002 and added a separately calculated child return, double counting the child's $45,000 wages. It also applied obsolete personal exemptions and a $1,000 CTC instead of the applicable 2026 rules." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model applied the expired-law personal-exemption regime and then added the child's separate tax to the parents' requested tax-unit output. The joint return excludes the child's wages and uses the $32,200 standard deduction, $2,000 non-itemizer charitable deduction, and $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model incorrectly placed the child's $45,000 wages in the joint tax unit, used personal exemptions, and treated the CTC as phased out. Joint AGI is $100,002, far below the CTC phaseout threshold, so the full $2,200 credit applies." +us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added the child's separate-return tax even though the requested variable is the parents' joint tax-unit liability. On the joint return it also used estimated deduction and bracket values and omitted the $2,000 non-itemizer charitable deduction." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model included the child's $45,000 wages in joint AGI, yielding about $145,002 rather than $100,002. The child's wages are excluded from the parents' return, whose taxable income is $65,802 after $34,200 of deductions." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The unexplained $14,677 estimate does not follow the traced joint-return computation: $100,002 AGI minus $34,200 of deductions yields $65,802 taxable income, then $7,400.24 of tax is reduced by the $2,200 CTC. Its magnitude reflects a calculation that failed to isolate and compute the parents' joint tax unit under those steps." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model included the child's $45,000 wages in the parents' joint AGI. Removing those wages while retaining the $32,200 standard deduction, $2,000 non-itemizer charitable deduction, and $2,200 CTC produces $5,200.24." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model correctly recognized both deductions and the $2,200 CTC but incorrectly included the child's $45,000 wages in joint AGI. Joint AGI is $100,002, so taxable income is $65,802 rather than $110,802." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used a $30,000 standard deduction instead of $32,200 and omitted the additional $2,000 charitable deduction for non-itemizers. Its $15,128.44 regular-tax figure also reflects inclusion of income outside the parents' $100,002 joint AGI." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model included the child's $45,000 wages in the joint tax calculation and invoked mortgage interest despite no interest-paid input. The parents' joint AGI is $100,002, and the applicable deduction is the $32,200 standard deduction plus $2,000 for non-itemizer charity." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model applied obsolete personal exemptions, old brackets, and a $1,000 CTC, then added the child's separate tax to the requested joint-tax-unit output. The applicable 2026 computation excludes the child's return and uses the $34,200 total deduction and $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model used the expired-law personal-exemption and 15% bracket structure and added the child's separate-return tax. The requested output is only the parents' joint tax unit under the applicable 2026 rules, including the $32,200 standard deduction, $2,000 non-itemizer charitable deduction, and $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model substantially reproduced the correct parents' computation but then added about $3,225 of tax from the child's separate return. The requested variable is attached to the parents' joint tax unit, so that separate liability is excluded; the charitable deduction also reduces taxable income rather than AGI." us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. -us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 wages in joint AGI and invented a $12,000 overtime deduction from the hourly schedule. The parents’ joint AGI is $100,002, and the trace’s deductions are the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction." -us,scenario_123,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model subtracted refundable credits when the requested measure is explicitly before refundable credits and incorrectly concluded the CTC erased all liability. Regular tax is $7,400.24 and the usable nonrefundable CTC is $2,200, leaving $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model included the child’s wages in joint AGI and invented $27,300 of mortgage interest from the mortgage balance and an assumed rate. The parents’ joint return has $100,002 of AGI and takes the $34,200 deduction allowed by the trace." -us,scenario_123,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly awarded both refundable CTC and EITC at roughly $145,002 of joint income. It treated the household as still inside the refundable CTC and EITC phaseout ranges, when the engine's phaseout calculation reduces both refundable components to zero for this one-child married-filing-jointly tax unit." -us,scenario_123,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model applied the refundable additional Child Tax Credit phase-in from earned income but skipped the high-income CTC phaseout interaction that eliminates the refundable CTC amount. At this joint income level, the one qualifying child does not generate a refundable CTC component after phaseout, so the refundable credit is zero rather than $1,500." -us,scenario_123,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model did not provide a parseable federal_refundable_credits value. Because the requested output was missing, the failure is a contract failure rather than a substantive tax calculation." +us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model included the child's $45,000 wages in joint gross income and invented a $12,000 overtime deduction from hours and straight-time wage information. The parents' joint AGI is $100,002, and no qualified-overtime amount was separately established in the traced computation." +us,scenario_123,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly used refundable credits to erase a quantity explicitly measured before refundable credits and treated the CTC as sufficient to eliminate the liability. Regular tax is $7,400.24 and the usable nonrefundable CTC is $2,200, leaving $5,200.24." +us,scenario_123,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model included the child's $45,000 wages in joint AGI and invented $27,300 of mortgage interest from the loan balance and an assumed rate. The parents' return has $100,002 AGI and takes the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction." +us,scenario_123,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model stacked itemized deductions on top of the standard deduction, invented mortgage interest, applied a QBI deduction to W-2 wages, and invented a dependent-care credit without care expenses. Standard and itemized deductions are alternatives, W-2 wages do not generate QBI, and only the $2,200 CTC reduces the $7,400.24 regular tax." +us,scenario_123,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model awarded a $1,600 refundable CTC without applying the 2026 joint-income phaseout that reduces the household’s CTC to zero. It also invented a residual $205 EITC even though approximately $145,002 of joint income is above the EITC phaseout range for one qualifying child." +us,scenario_123,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated one child under 17 and sufficient earned income as enough to produce a $1,500 Additional Child Tax Credit. It omitted the 2026 CTC income phaseout, which eliminates the credit at this household’s approximately $145,002 joint income before any refundable amount remains." +us,scenario_123,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model used a $1,700 enhanced Additional Child Tax Credit cap and treated sufficient earnings as guaranteeing the full refund. It failed to apply the 2026 CTC income phaseout, which reduces the parents’ credit to zero at approximately $145,002 of joint income." +us,scenario_123,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for federal_refundable_credits, violating the required structured-output contract." +us,scenario_123,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly treated the child’s $45,000 of wages as making a $2,000 Additional Child Tax Credit fully refundable and used an inapplicable enhanced cap. Refundability does not bypass the 2026 CTC joint-income phaseout, which eliminates the household’s credit at approximately $145,002 of income." us,scenario_123,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_123,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. +us,scenario_123,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable head_medicaid_eligible value or explanation, violating the required function-output contract." +us,scenario_123,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly asserted that the household's MAGI was below Pennsylvania's adult Medicaid threshold, even though the trace places MAGI at 5.31 times FPL. It also treated being under age 65 as sufficient categorical status, while the head qualifies through no Medicaid pathway." us,scenario_123,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_123,local_income_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model treated every Pennsylvania household as subject to Philadelphia wage tax and applied the Philadelphia resident wage tax rate to the head's and child's wages. It inferred Philadelphia residence from the state alone, then introduced an unsupported exemption adjustment for the child instead of recognizing that no Philadelphia, NYC, Kansas City, or St. Louis locality was specified." -us,scenario_123,local_income_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model assumed Philadelphia was the relevant Pennsylvania locality and applied a Philadelphia wage-tax rate to $145,000 of wages even though the facts give only Pennsylvania residence. Its submitted $3,830.90 is internally inconsistent with its own $5,437.50 and $5,495.50 calculations, but the substantive error is the same unsupported Philadelphia wage-tax assignment." -us,scenario_123,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model's written computation correctly totaled Social Security, Medicare, and Pennsylvania employee unemployment contributions to $11,194, but it submitted $11,663.35 instead. Its numeric output contradicts its own completed arithmetic." -us,scenario_123,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly computed federal FICA as $11,092.50, then incorrectly added $648.89 of Additional Medicare Tax despite stating that no wages exceeded the applicable threshold. It also denied Pennsylvania's mandatory employee unemployment-compensation contribution, which adds $101.50." -us,scenario_123,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model calculated Pennsylvania's 0.07% employee unemployment-compensation contribution as $101.50 but then replaced that component with zero in its final sum. Adding that mandatory state payroll tax to $11,092.50 of federal FICA yields $11,194." -us,scenario_123,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model explicitly calculated each federal and Pennsylvania component and correctly recomputed their total as $11,194, but submitted $11,097.50. The submitted value is an arithmetic/output-transfer error that contradicts its reasoning." -us,scenario_123,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model stopped at the $11,092.50 federal Social Security and Medicare subtotal. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution on $145,000 of wages, equal to $101.50." -us,scenario_123,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model incorrectly capped Pennsylvania's employee unemployment-compensation contribution at $10,000 of wages per worker, producing only $14. The 0.07% employee contribution applies to the full $145,000 wage base here, producing $101.50 and a total payroll tax of $11,194." -us,scenario_123,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model incorrectly asserted that Pennsylvania has no mandatory employee unemployment or state payroll withholding. Pennsylvania's 0.07% employee unemployment-compensation contribution adds $101.50 to the $11,092.50 federal FICA subtotal." -us,scenario_123,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model computed only employee Social Security and Medicare taxes, totaling $11,092.50. It omitted Pennsylvania's mandatory employee unemployment-compensation contribution of $101.50." -us,scenario_123,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model said it applied 7.65% FICA to $145,000, but that multiplication equals $11,092.50 rather than $11,108. It also omitted the $101.50 Pennsylvania employee unemployment-compensation contribution, so the complete total is $11,194." -us,scenario_123,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly asserted that Pennsylvania has no mandatory employee-side state payroll tax. The state's 0.07% employee unemployment-compensation contribution on $145,000 adds $101.50 to its $11,092.50 federal FICA subtotal." -us,scenario_123,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model described the correct federal FICA bases and correctly ruled out Additional Medicare Tax, but $12,366 does not follow from those stated components. Federal FICA is $11,092.50 and Pennsylvania's employee unemployment contribution is $101.50, producing $11,194." -us,scenario_123,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's $15,399 does not equal Social Security at 6.2% plus Medicare at 1.45% on the listed $145,000 of wages. Those federal components total $11,092.50, and the required $101.50 Pennsylvania employee unemployment contribution brings the result to $11,194." -us,scenario_123,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model stopped after computing the $11,092.50 federal FICA subtotal. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution on the two workers' $145,000 of wages, equal to $101.50." -us,scenario_123,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model's stated calculation of 7.65% on $100,000 and $45,000 equals $11,092.50, not $11,107.50. It also omitted Pennsylvania's $101.50 mandatory employee unemployment-compensation contribution, which makes the complete total $11,194." -us,scenario_123,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model correctly obtained $11,092.50 from federal Social Security and Medicare taxes but incorrectly asserted that Pennsylvania has no mandatory employee payroll tax. Pennsylvania's employee unemployment-compensation contribution adds $101.50." -us,scenario_123,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model correctly calculated the federal FICA subtotal of $11,092.50 but incorrectly excluded all mandatory state employee payroll taxes. Pennsylvania's 0.07% employee unemployment-compensation contribution on $145,000 is $101.50." -us,scenario_123,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no payroll-tax value or explanation. It therefore failed the required structured-output contract. -us,scenario_123,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model acknowledged that the listed wages generate employee Social Security and Medicare taxes but then set the result to zero without performing the computation. The wages produce $11,092.50 of federal FICA plus $101.50 of Pennsylvania employee unemployment contributions, totaling $11,194." -us,scenario_123,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model correctly computed $11,092.50 of federal FICA but incorrectly stated that Pennsylvania has no employee-side state payroll tax. Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution adds $101.50." +us,scenario_123,local_income_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model inferred Philadelphia residence solely from the household's Pennsylvania state and applied Philadelphia's wage tax despite the instruction to treat unlisted locality facts as false. Its unexplained reduction from the computed $5,613.24 to $3,545 also applies no identified exemption or valid computation step." +us,scenario_123,local_income_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated Philadelphia as the default Pennsylvania locality and taxed both workers' wages even though Philadelphia residence or employment was never listed. It also submitted $3,830.90 after its own calculations produced $5,437.50 or $5,495.50, so the final number does not follow from its stated rate or tax base." +us,scenario_123,payroll_tax,claude-fable-5,llm_error,other,False,"The model explicitly calculated all three components and their correct $11,194 sum, then submitted $11,663.35 instead. Its error is a final-answer transcription failure disconnected from its own computation." +us,scenario_123,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly found $11,092.50 of federal FICA but omitted Pennsylvania's $101.50 employee unemployment-compensation contribution. It then invented $648.89 of Additional Medicare Tax despite stating that neither the individual nor joint threshold was exceeded." +us,scenario_123,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model calculated Pennsylvania's 0.07% employee unemployment-compensation contribution as $101.50, then replaced that component with zero in the final sum. Adding it to the $11,092.50 federal FICA total yields $11,194." +us,scenario_123,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model correctly computed $7,720 for the head and $3,474 for the child, explicitly summed them to $11,194, and then submitted $11,097.50. The submitted value is a transcription or arithmetic-output error unsupported by its derivation." +us,scenario_123,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model stopped after computing $11,092.50 of employee Social Security and Medicare tax. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution of $101.50 on the workers' $145,000 of wages." +us,scenario_123,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly capped Pennsylvania's employee unemployment-compensation contribution at $10,000 of wages per worker, producing only $14. The 0.07% employee contribution applies to the full $145,000 of combined wages here, producing $101.50 and a total payroll tax of $11,194." +us,scenario_123,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model incorrectly asserted that Pennsylvania has no mandatory employee unemployment-insurance withholding. It therefore omitted the state's 0.07% employee contribution of $101.50 from the otherwise correct $11,092.50 federal FICA calculation." +us,scenario_123,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model calculated only employee Social Security and Medicare taxes. It omitted Pennsylvania's mandatory employee unemployment-compensation contribution of $101.50, which raises the total from $11,092.50 to $11,194." +us,scenario_123,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"Applying the stated 7.65% FICA rate to $145,000 yields $11,092.50, not $11,108. The model both miscomputed its stated FICA calculation and failed to add the actual $101.50 Pennsylvania employee unemployment-compensation contribution." +us,scenario_123,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly treated Pennsylvania mandatory employee-side payroll tax as zero. Pennsylvania's 0.07% employee unemployment-compensation contribution adds $101.50 to its correctly calculated $11,092.50 of federal FICA." +us,scenario_123,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The stated Social Security and Medicare calculation on $145,000 of wages yields $11,092.50, not $12,366. The model also assumed away Pennsylvania's $101.50 employee unemployment-compensation contribution, so its submitted number follows neither the federal arithmetic nor the complete payroll-tax definition." +us,scenario_123,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"Employee Social Security and Medicare tax on $145,000 equals $11,092.50, not $15,399. The model misapplied or miscomputed the federal rates and also omitted Pennsylvania's $101.50 employee unemployment-compensation contribution." +us,scenario_123,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model correctly computed $11,092.50 of federal FICA but stopped there. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution of $101.50." +us,scenario_123,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"FICA at 7.65% on the two workers' $145,000 of wages is $11,092.50, not $11,107.50. The model also omitted the separately required Pennsylvania employee unemployment-compensation contribution of $101.50." +us,scenario_123,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model correctly calculated $11,092.50 of employee Social Security and Medicare tax but incorrectly set mandatory Pennsylvania employee payroll tax to zero. The 0.07% unemployment-compensation contribution adds $101.50." +us,scenario_123,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,The model incorrectly concluded that Pennsylvania imposed no mandatory employee state payroll tax. It omitted the 0.07% employee unemployment-compensation contribution of $101.50 from its otherwise correct federal FICA total. +us,scenario_123,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model correctly derived $11,092.50 of federal Social Security and Medicare tax but set state mandatory employee payroll taxes to zero. Pennsylvania's employee unemployment-compensation contribution is $101.50 on the combined wages." +us,scenario_123,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll-tax value or explanation. It therefore failed the required structured-output contract rather than completing a substantive calculation. +us,scenario_123,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model acknowledged that the listed wages generate employee Social Security and Medicare liabilities but discarded that calculation and submitted zero. The wage inputs produce $11,092.50 of federal FICA plus $101.50 of Pennsylvania employee unemployment-compensation tax." +us,scenario_123,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly asserted that Pennsylvania has no employee-side state payroll tax. Its federal calculation is $11,092.50, and the omitted 0.07% employee unemployment-compensation contribution adds $101.50." +us,scenario_123,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model mislabeled the head's total FICA as Social Security, counted Medicare a second time, and imposed $270 of Additional Medicare Tax even though the stated wages do not exceed the applicable threshold. It then applied an unsupported rounding adjustment instead of summing $8,990 of Social Security, $2,102.50 of Medicare, and $101.50 of Pennsylvania employee unemployment-compensation tax." us,scenario_123,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,spouse_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_123,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. +us,scenario_123,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no spouse_medicaid_eligible output, violating the required submission contract and omitting the engine-derived result of 0." +us,scenario_123,spouse_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly stated that the spouse's MAGI was below Pennsylvania's adult Medicaid threshold. The spouse's MAGI is 5.31 times FPL, far above the applicable income limits, and the spouse qualifies through no other Medicaid category." us,scenario_123,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,spouse_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model added the child's separately assessed $45,000 of wages to the parents' state-tax output. It also submitted $4,453.06 after correctly computing its own mistaken combined-base result as $4,451.56." -us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model both included the child's separate wages and invented Pennsylvania standard deductions for the parents and child. Pennsylvania applies no such standard deductions here; the requested output taxes only the parents' $100,002 base at 3.07%." -us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model incorrectly included the child's $45,000 of wages in the parents' Pennsylvania tax-unit output. It then submitted $4,452.08 even though its stated multiplication on the mistaken $145,002 base produced $4,451.56." -us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model treated every household member's income as one Pennsylvania filing unit and included the child's $45,000. It also submitted $4,452.61 instead of the $4,451.56 generated by its own mistaken base." -us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model added $1,381.50 of tax on the child's separately filed $45,000 of wages to the requested parents' tax-unit result. The requested output is limited to 3.07% of the parents' $100,002." -us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,The model explicitly recognized that the child files a separate Pennsylvania return but then overrode that rule because the child lives in the same household. Household membership does not merge the child's separate filing unit into the parents' state-income-tax output. -us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model identified the correct $100,002 taxable base and correctly calculated $3,070.06, then replaced that exact result with an unsupported approximation of $3,060. Pennsylvania's compensation and interest treatment supplies no $10.06 adjustment." -us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model added tax on the child's $45,000 of separately assessed wages to the parents' state-tax output. It also omitted the parents' $2 of taxable interest, which contributes $0.06 to the correct $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model combined the child's $45,000 of wages with the head-and-spouse filing unit. The requested Pennsylvania output uses only the parents' $100,002 of taxable wages and interest." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used a rounded 3% rate instead of Pennsylvania's 3.07% rate and applied it to the full household's $145,000 of wages. The correct computation applies 3.07% to the parents' $100,002 taxable income." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model treated the full household's $145,002 as the taxable income of one Pennsylvania filing unit. The child's wages belong to a separate filing unit, leaving $100,002 in the requested tax unit." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model correctly computed the parents' tax at about $3,070 but then added approximately $1,382 of tax from the child's separate return. That separate liability is not part of this requested tax-unit output." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model applied 3.07% to income from all household members, including the child's $45,000. The relevant filing unit contains only the head and spouse and has $100,002 of taxable income." -us,scenario_123,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model separately calculated the parents' correct $3,070.06 and the child's $1,381.50, then incorrectly summed both returns into one output. The requested value is the parents' tax-unit liability only." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The submitted $4,807 does not follow from Pennsylvania's 3.07% rate applied to either the correct $100,002 base or the model's implied full-household $145,002 base. The correct multiplication is $100,002 × 0.0307 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model used an unidentified itemized-deduction calculation even though Pennsylvania does not use the listed federal-style medical and real-estate deductions to determine this taxable base. Its $9,055 also does not follow from the 3.07% rate applied to the parents' $100,002." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model included the child's $45,000 of wages in the parents' Pennsylvania filing unit. Excluding that separate filing unit leaves $100,002 taxed at 3.07%." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model merged the child's wages into the parents' taxable base and also misstated the combined income by describing $145,002 of wages plus another $2 of interest. Its submitted amount does not equal 3.07% of either that described base or the $145,002 full-household base." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model taxed the full household's $145,002 rather than the head-and-spouse tax unit's $100,002. The child's $45,000 belongs to a separate Pennsylvania filing unit." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,The model treated household membership as a single Pennsylvania filing unit and included the child's earnings. The requested tax-unit base excludes those separately filed wages. -us,scenario_123,state_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model used approximately $145,000 of combined household income, thereby including the child's separate wages. Pennsylvania tax for the requested unit is 3.07% of the parents' $100,002, with no relevant deductions." -us,scenario_123,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model applied the Pennsylvania rate to $145,002 by combining the child's separate earnings with the parents' income. Only the parents' $100,002 belongs in this output." -us,scenario_123,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model explicitly included the child's $45,000 in the parents' taxable base and rounded the resulting separate-return total. The requested output excludes the child's filing unit and equals $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The submitted $2,939.86 corresponds to taxing only about $95,761 rather than the traced Pennsylvania taxable income of $100,002. The model therefore imposed an unsupported reduction of roughly $4,241 before applying the 3.07% rate." -us,scenario_123,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model included the child's separately filed $45,000 of wages in the requested Pennsylvania tax unit. The correct taxable base for this output is $100,002." -us,scenario_123,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model stated that Pennsylvania liability is positive but nevertheless submitted zero without performing the required computation. Applying 3.07% to the parents' $100,002 taxable income yields $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly combined the child's $45,000 of separately filed wages with the parents' taxable income. It also rounded $145,002 × 3.07% incorrectly to $4,451.57 rather than $4,451.56, but the controlling error is use of the wrong filing unit." +us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model added the child’s separate $45,000 Pennsylvania filing-unit income to the parents’ $100,002 tax base. It also submitted $4,453.06 despite correctly calculating its own erroneous combined-base result as $4,451.56." +us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model both included the child’s separately taxed wages in the parents’ Pennsylvania calculation and invented Pennsylvania standard deductions of $15,000 and $8,500. Pennsylvania does not use those federal-style deductions; the relevant parental tax base is $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model improperly included the child’s $45,000 wages in the benchmark’s parental Pennsylvania tax unit, inflating taxable income from $100,002 to $145,002. It then submitted $4,452.08 even though its stated multiplication produced $4,451.56." +us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model treated all household members as one Pennsylvania filing unit and included the child’s separately taxed $45,000 of wages. It also submitted $4,452.61 instead of the $4,451.56 produced by its stated erroneous base." +us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model added the child’s separate $1,381.50 Pennsylvania liability to the parents’ joint-return liability. The requested tax-unit output includes only the parents’ $100,002 base, which yields $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model explicitly identified that the child files a separate Pennsylvania return but then reversed that conclusion because everyone was described as one household. Household co-residence does not merge the child into the parents’ state income-tax filing unit, so the child’s $45,000 must be excluded." +us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model derived the correct $100,002 Pennsylvania taxable income and the exact $3,070.06 tax, then replaced it with an unsupported rounded amount of $3,060. Pennsylvania’s treatment of compensation and interest supplies no $10.06 adjustment." +us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model added $1,381.50 of tax on the child’s separately filed $45,000 wages to the parents’ liability. It also omitted the parents’ $2 of taxable interest, which contributes $0.06 to their correct $3,070.06 liability." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model combined the child’s $45,000 wages with the parents’ $100,002 Pennsylvania tax base. The child belongs to a separate state filing unit, leaving $100,002 subject to the 3.07% rate in this output." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used a 3% Pennsylvania rate instead of 3.07% and applied it to the full household’s $145,000 of wages. The benchmark requires the 3.07% rate on only the parents’ $100,002 taxable income." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model treated $145,002 across all household members as one Pennsylvania taxable-income base. The child’s $45,000 belongs to a separate filing unit and is excluded from the parents’ $100,002 base." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model correctly computed the parents’ liability at about $3,070 but then added approximately $1,382 for the child’s separate Pennsylvania return. The requested output stops at the parents’ filing unit and therefore excludes the child’s liability." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model applied Pennsylvania’s rate to $145,002 by merging the child’s wages into the parents’ filing unit. Excluding the child’s separate $45,000 leaves the correct $100,002 base." +us,scenario_123,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model separately calculated the parents’ correct $3,070.06 liability and the child’s $1,381.50 liability, then summed both. The benchmark state-tax output corresponds to the parents’ joint tax unit, not the sum of separate returns within the residence." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The submitted $4,807 corresponds to applying 3.07% to roughly $156,580, a tax base unsupported by any listed Pennsylvania-taxable income. The relevant parental wages and interest total $100,002, producing $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model imported federal-style itemized-deduction reasoning into Pennsylvania’s class-based income tax and produced a liability that does not follow from the listed income. Pennsylvania taxes the parents’ $100,000 of wages and $2 of interest directly at 3.07% for $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 wages in the parents’ Pennsylvania taxable-income base. Those wages belong to the child’s separate filing unit, so the applicable base is $100,002 rather than $145,002." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model merged the child’s wages into the parental filing unit by using a $145,002 household base instead of $100,002. Its submitted $4,455.06 also does not equal 3.07% of the base it stated." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model taxed $145,002 by combining the child’s separate earnings with the parents’ income. The child’s $45,000 is outside the parental Pennsylvania filing unit represented by this output." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model treated household membership as Pennsylvania joint-return membership and taxed all $145,002 together. Only the parents’ $100,002 is included in the relevant tax unit." +us,scenario_123,state_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model’s roughly $145,000 combined base includes the child’s separately taxed $45,000 of wages. The parental Pennsylvania tax base is $100,002, and no listed deduction changes it." +us,scenario_123,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model combined the child’s wages with the parents’ compensation and interest to form a $145,002 base. The child files separately for Pennsylvania purposes, so the benchmark uses only $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model expressly included the child’s $45,000 wages in the parents’ Pennsylvania calculation. Those wages belong to a separate filing unit and must not be added to the parents’ $100,002 base." +us,scenario_123,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model summed tax across two Pennsylvania filing units: $100,002 for the parents and $45,000 for the child. The requested output represents the parents’ tax unit alone, so the child’s separate liability is excluded." +us,scenario_123,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The $2,939.86 answer corresponds to taxing approximately $95,761 at 3.07%, an unsupported reduction from the parents’ $100,002 Pennsylvania taxable income. Pennsylvania’s calculation leaves that $100,002 base intact and yields $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model used $145,002 by adding the child’s wages to the parents’ Pennsylvania income. The child’s $45,000 belongs to a separate filing unit, leaving the parental base at $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model stated that Pennsylvania tax liability is positive but submitted zero, contradicting its own reasoning. Applying 3.07% to the parents’ $100,002 of taxable wages and interest yields $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model improperly included the child’s $45,000 wages in the parents’ Pennsylvania taxable-income base. It also misstated the arithmetic on its erroneous $145,002 base, which produces $4,451.56 when rounded to cents, not $4,451.57." +us,scenario_123,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model substituted federal taxable income for Pennsylvania taxable income even though Pennsylvania does not begin with that federal post-deduction figure. It then submitted $2,565.30 despite stating that its own calculation produced $2,073.02; the correct Pennsylvania base is the parents’ $100,002." us,scenario_123,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. diff --git a/app/src/components/Hero.tsx b/app/src/components/Hero.tsx index da9f1fc..9470972 100644 --- a/app/src/components/Hero.tsx +++ b/app/src/components/Hero.tsx @@ -2,7 +2,7 @@ import { DEFAULT_VERSION_ID } from "../lib/dataVersionsRuntime"; import type { BenchData, CountryCode } from "../types"; import SiteHeader, { type HeaderNavItem } from "./SiteHeader"; -const SNAPSHOT_DATE_LABEL = "Snapshot 2026-07-24"; +const SNAPSHOT_DATE_LABEL = "Snapshot 2026-08-05"; export default function Hero({ selectedView, diff --git a/app/src/components/ProviderMark.tsx b/app/src/components/ProviderMark.tsx index 9e81fc4..1bf65ea 100644 --- a/app/src/components/ProviderMark.tsx +++ b/app/src/components/ProviderMark.tsx @@ -11,6 +11,38 @@ import { } from "@lobehub/icons"; import { PROVIDER_LABELS, type ProviderKey } from "../modelMeta"; +// Thinking Machines Lab has no @lobehub/icons mark yet (checked 5.15.0), so +// this embeds the company's own 32px favicon (thinkingmachines.ai — their +// square mark) as an SVG mask filled with currentColor, keeping the canonical +// geometry while staying legible in both themes. Swap for the @lobehub icon +// once one ships. +const TML_FAVICON = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAABEUlEQVRYR2NkAIJnz54FMTAyNQCZWkDMDBKjIfgLNPsaw/9/DVJSUusYn718GcTw7/9qoCATDS3FZvQ/BibGUMbnL15c/P+fQY/OloOtY2RkuMT47PmLP3QIdlz++wtywP+B8D3MTqwO+PHjB8PylWsYnr94SRW3SUqIM0SGhzBwcHBgmIfVAfMXLWVYs24DVSyHGRISFMCQGBdNnAPau/oYjhw7TlUH2FhZMlSWFY06YDQERkNgNARGQ2A0BEZDYDQEhkgIDHiT7Du0UfqCSo1SCWCjNIqURilVG4MEDBv4jsnAd82eATunjAPUOf0P7JyCogjSPWdsZGRk0vz//z9Nu+eMjIx/Gf7/v/Yf2j0HAN9HH5MVKknXAAAAAElFTkSuQmCC"; + +function ThinkingMachinesMark({ size }: { size: number }) { + return ( + + ); +} + export default function ProviderMark({ provider, size = 14, @@ -49,6 +81,7 @@ export default function ProviderMark({ {provider === "minimax" &&