{
  "title": "Tax schema comparison: printed-summary fidelity",
  "scope": "Targeted experiment on five previously examined invoices, not production accuracy.",
  "method": {
    "cases": 5,
    "repeats_per_variant_per_case": 5,
    "variants": 3,
    "requests": 75,
    "completed": 74,
    "model": "Qwen 3.7 Flash",
    "temperature": 0,
    "reasoning": "disabled",
    "strict_schema": true,
    "arm_order": "rotated",
    "concurrency": 3,
    "retries": 0
  },
  "criterion": "Capture exactly the printed invoice-summary tax figures without inventing totals or moving line taxes into the summary.",
  "controls": "Same source inputs and provider settings across arms; equivalent tax-location examples translated into each format.",
  "variants": {
    "baseline": "Earlier experimental tax list with balanced examples, not the production baseline.",
    "structured": "Separate breakdown and total with positional item arrays.",
    "named": "Same separation with named item fields."
  },
  "results": [
    {
      "variant": "baseline",
      "requests": 25,
      "schema_valid": 25,
      "faithful_printed_summary": 18
    },
    {
      "variant": "structured",
      "requests": 25,
      "schema_valid": 25,
      "faithful_printed_summary": 10
    },
    {
      "variant": "named",
      "requests": 25,
      "schema_valid": 24,
      "faithful_printed_summary": 7
    }
  ],
  "decision": "Reject these candidates; keep production unchanged.",
  "limitations": [
    "Five familiar invoices from two suppliers; no unseen-layout evaluation.",
    "Schema organization and corresponding instructions/examples changed together.",
    "Named variant had one timeout; its summary score is 7/24 completed responses.",
    "No claim of overall extraction accuracy or commercial adoption."
  ],
  "provenance": "Aggregated from retained model responses using the original local grader; scores reproduced during portfolio preparation. Underlying customer documents and provider requests are not published.",
  "scored_attempts": [
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "baseline",
      "schema_valid": true,
      "faithful_printed_summary": true
    },
    {
      "variant": "named",
      "schema_valid": false,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "named",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    },
    {
      "variant": "structured",
      "schema_valid": true,
      "faithful_printed_summary": false
    }
  ]
}