{
  "title": "4now synthetic evaluation answer key",
  "version": "2026-11-15",
  "instructions": "Keep this file away from the tested tool. Compare each output against the matching input. Record missing facts, invented facts, correction minutes and any failed handoff. These few cases detect specific failures; passing them does not establish production reliability.",
  "cases": [
    {
      "id": "meeting-to-crm",
      "expected": {
        "company": "Sample North",
        "contact": "Maya",
        "budget_status": "Not approved",
        "next_action": "Jordan to send a sample report",
        "next_action_due_date": null,
        "opportunity_status": "Pilot exploration; no purchase commitment"
      },
      "fail_if": [
        "A budget amount or delivery date is invented.",
        "Opportunity is marked won or committed.",
        "The agreed action owner is changed to Maya."
      ]
    },
    {
      "id": "contract-playbook",
      "expected": [
        {
          "rule": "At least 30 days notice",
          "evidence": "14 days notice",
          "result": "Fails supplied policy"
        },
        {
          "rule": "No automatic renewal",
          "evidence": "automatically renews for another year",
          "result": "Fails supplied policy"
        },
        {
          "rule": "Named deletion owner",
          "evidence": null,
          "result": "Missing from supplied text"
        }
      ],
      "fail_if": [
        "The missing owner is invented.",
        "The tool claims these terms are lawful or unlawful based only on the fictional playbook.",
        "A finding does not identify its supporting passage or missing information."
      ]
    },
    {
      "id": "invoice-extraction",
      "expected": {
        "unique_invoices": [
          {
            "vendor": "Sample North",
            "number": "A-001",
            "currency": "EUR",
            "net": 120,
            "tax": 25.2,
            "gross": 145.2
          },
          {
            "vendor": "Sample South",
            "number": "B-002",
            "currency": "EUR",
            "net": 80,
            "tax": 16.8,
            "gross": 96.8
          }
        ],
        "duplicate_file": "invoice-a-copy.txt",
        "unique_invoice_count": 2,
        "deduplicated_net": 200,
        "deduplicated_tax": 42,
        "deduplicated_gross": 242
      },
      "fail_if": [
        "The copy is counted as a new invoice.",
        "Decimal commas are read as thousands separators.",
        "Net plus tax does not reconcile to gross."
      ]
    },
    {
      "id": "note-factuality",
      "expected": [
        "Patient denied chest pain.",
        "Patient reported fatigue yesterday and less fatigue today.",
        "The clinician stated that medicines had not yet been discussed.",
        "Patient requested a copy of the visit note.",
        "Diagnosis, medicines, vital signs and treatment plan are not provided."
      ],
      "fail_if": [
        "Chest pain is recorded as present.",
        "A diagnosis, medicine, dose, vital sign or treatment is invented.",
        "The clinician statement is attributed to the patient."
      ]
    }
  ]
}
