{
  "summary": {
    "date": "2026-10-31",
    "method": "One run per format per original synthetic fixture; no failed run discarded or retried. Automatic quotation matching is not answer-accuracy grading.",
    "source_documents": [
      {
        "id": "electricity",
        "file": "electricity.pdf",
        "pages": 3,
        "kind": "selectable text",
        "license": "CC0-1.0",
        "synthetic": true
      },
      {
        "id": "water-cycle",
        "file": "water-cycle.pdf",
        "pages": 3,
        "kind": "selectable text",
        "license": "CC0-1.0",
        "synthetic": true
      },
      {
        "id": "economics",
        "file": "economics.pdf",
        "pages": 3,
        "kind": "selectable text",
        "license": "CC0-1.0",
        "synthetic": true
      },
      {
        "id": "water-cycle-scan",
        "file": "water-cycle-scan.pdf",
        "pages": 3,
        "kind": "image-only scan",
        "original_text_source": "water-cycle.pdf",
        "license": "CC0-1.0",
        "synthetic": true
      }
    ],
    "cases": [
      {
        "document": "electricity",
        "format": "mcq",
        "requested": 5,
        "started_at": "2026-10-31T05:35:38.698541+00:00",
        "http_status": 200,
        "returned": 5,
        "source_pages": [
          1,
          2,
          3
        ],
        "original_quote_matches": 5,
        "verified_by_service": 5,
        "format_valid": true,
        "answer_positions": [
          2,
          1,
          3,
          4,
          3
        ],
        "full_page_coverage": true,
        "seconds": 9.8
      },
      {
        "document": "electricity",
        "format": "true_false",
        "requested": 5,
        "started_at": "2026-10-31T05:35:38.698541+00:00",
        "http_status": 200,
        "returned": 5,
        "source_pages": [
          1,
          2,
          3
        ],
        "original_quote_matches": 5,
        "verified_by_service": 5,
        "format_valid": true,
        "answer_positions": [
          2,
          1,
          2,
          1,
          1
        ],
        "full_page_coverage": true,
        "seconds": 8.6
      },
      {
        "document": "electricity",
        "format": "short_answer",
        "requested": 5,
        "started_at": "2026-10-31T05:35:47.273852+00:00",
        "http_status": 200,
        "returned": 5,
        "source_pages": [
          1,
          2,
          3
        ],
        "original_quote_matches": 5,
        "verified_by_service": 5,
        "format_valid": true,
        "answer_positions": [],
        "full_page_coverage": true,
        "seconds": 6.3
      },
      {
        "document": "water-cycle",
        "format": "mcq",
        "requested": 5,
        "started_at": "2026-10-31T05:35:55.661288+00:00",
        "http_status": 200,
        "returned": 5,
        "source_pages": [
          1,
          2,
          3
        ],
        "original_quote_matches": 5,
        "verified_by_service": 5,
        "format_valid": true,
        "answer_positions": [
          3,
          2,
          4,
          1,
          3
        ],
        "full_page_coverage": true,
        "seconds": 8.9
      },
      {
        "document": "water-cycle",
        "format": "true_false",
        "requested": 5,
        "started_at": "2026-10-31T05:35:55.662277+00:00",
        "http_status": 503,
        "error": "{\"detail\":\"Quiz generation temporarily unavailable\"}",
        "seconds": 2.9
      },
      {
        "document": "water-cycle",
        "format": "short_answer",
        "requested": 5,
        "started_at": "2026-10-31T05:35:58.608779+00:00",
        "http_status": 503,
        "error": "{\"detail\":\"Quiz generation temporarily unavailable\"}",
        "seconds": 2.1
      },
      {
        "document": "economics",
        "format": "mcq",
        "requested": 5,
        "started_at": "2026-10-31T05:36:05.328933+00:00",
        "http_status": 503,
        "error": "{\"detail\":\"Quiz generation temporarily unavailable\"}",
        "seconds": 3.5
      },
      {
        "document": "economics",
        "format": "true_false",
        "requested": 5,
        "started_at": "2026-10-31T05:36:05.329931+00:00",
        "http_status": 503,
        "error": "{\"detail\":\"Quiz generation temporarily unavailable\"}",
        "seconds": 2.9
      },
      {
        "document": "economics",
        "format": "short_answer",
        "requested": 5,
        "started_at": "2026-10-31T05:36:08.209118+00:00",
        "http_status": 503,
        "error": "{\"detail\":\"Quiz generation temporarily unavailable\"}",
        "seconds": 2.3
      },
      {
        "document": "water-cycle-scan",
        "format": "mcq",
        "requested": 5,
        "started_at": "2026-10-31T05:36:11.481268+00:00",
        "http_status": 200,
        "returned": 5,
        "source_pages": [
          1,
          2,
          3
        ],
        "original_quote_matches": 5,
        "verified_by_service": 5,
        "format_valid": true,
        "answer_positions": [
          3,
          2,
          1,
          4,
          2
        ],
        "full_page_coverage": true,
        "seconds": 20.5
      },
      {
        "document": "water-cycle-scan",
        "format": "true_false",
        "requested": 5,
        "started_at": "2026-10-31T05:36:31.944029+00:00",
        "http_status": 503,
        "error": "{\"detail\":\"Quiz generation temporarily unavailable\"}",
        "seconds": 12.2
      },
      {
        "document": "water-cycle-scan",
        "format": "short_answer",
        "requested": 5,
        "started_at": "2026-10-31T05:36:44.123599+00:00",
        "http_status": 200,
        "returned": 5,
        "source_pages": [
          1,
          2,
          3
        ],
        "original_quote_matches": 5,
        "verified_by_service": 5,
        "format_valid": true,
        "answer_positions": [],
        "full_page_coverage": true,
        "seconds": 18.1
      }
    ],
    "negative_test": {
      "case": "malformed PDF rejection",
      "http_status": 200,
      "response": "{\"error\":\"OCR failed. Please try a clearer PDF scan.\"}"
    },
    "requested_total": 60,
    "returned_total": 30,
    "original_quote_matches": 30,
    "failed_cases": 6,
    "fully_covered_cases": 6,
    "limitations": [
      "Small, authored synthetic fixtures, not real student documents or independent evaluation.",
      "Clean English scans only; no handwriting, poor scans, charts or complex formula validation.",
      "Quoted text matches do not certify the source, generated answer or pedagogical quality.",
      "Model outputs and runtime can vary; one trial does not estimate reliability."
    ]
  },
  "attempts": [
    {
      "document": "economics",
      "format": "mcq",
      "requested": 5,
      "started_at": "2026-10-31T05:36:05.328933+00:00",
      "http_status": 503,
      "error": "{\"detail\":\"Quiz generation temporarily unavailable\"}",
      "seconds": 3.5
    },
    {
      "document": "economics",
      "format": "short_answer",
      "requested": 5,
      "started_at": "2026-10-31T05:36:08.209118+00:00",
      "http_status": 503,
      "error": "{\"detail\":\"Quiz generation temporarily unavailable\"}",
      "seconds": 2.3
    },
    {
      "document": "economics",
      "format": "true_false",
      "requested": 5,
      "started_at": "2026-10-31T05:36:05.329931+00:00",
      "http_status": 503,
      "error": "{\"detail\":\"Quiz generation temporarily unavailable\"}",
      "seconds": 2.9
    },
    {
      "document": "electricity",
      "format": "mcq",
      "requested": 5,
      "started_at": "2026-10-31T05:35:38.698541+00:00",
      "http_status": 200,
      "returned": 5,
      "source_pages": [
        1,
        2,
        3
      ],
      "original_quote_matches": 5,
      "verified_by_service": 5,
      "format_valid": true,
      "answer_positions": [
        2,
        1,
        3,
        4,
        3
      ],
      "quiz": [
        {
          "question": "What is the unit used to measure electric charge?",
          "options": [
            "volts",
            "coulombs",
            "amperes",
            "ohms"
          ],
          "answer": "coulombs",
          "explanation": "The source states \"Electric charge is measured in coulombs,\" directly identifying coulombs as the unit for charge.",
          "source_page": 1,
          "source_quote": "Electric charge is measured in coulombs.",
          "proof_verified": true
        },
        {
          "question": "What does a voltmeter measure and how is it connected?",
          "options": [
            "measures potential difference, connected in parallel",
            "measures resistance, connected in series",
            "measures current, connected in series",
            "measures power, connected in parallel"
          ],
          "answer": "measures potential difference, connected in parallel",
          "explanation": "The passage notes \"A voltmeter measures potential difference and is connected in parallel across a component,\" matching the correct option.",
          "source_page": 2,
          "source_quote": "A voltmeter measures potential difference and is connected in parallel across a component.",
          "proof_verified": true
        },
        {
          "question": "In a series circuit, how does the total resistance relate to individual resistances?",
          "options": [
            "average of individual resistances",
            "difference of individual resistances",
            "sum of individual resistances",
            "product of individual resistances"
          ],
          "answer": "sum of individual resistances",
          "explanation": "The passage notes \"the total resistance is the sum of the individual resistances,\" confirming the sum relationship.",
          "source_page": 3,
          "source_quote": "the total resistance is the sum of the individual resistances.",
          "proof_verified": true
        },
        {
          "question": "How is an ammeter connected in a circuit?",
          "options": [
            "in parallel with the component",
            "between battery terminals",
            "across the power source",
            "in series with the component"
          ],
          "answer": "in series with the component",
          "explanation": "The text says \"An ammeter measures current and is connected in series with the component,\" confirming series connection.",
          "source_page": 1,
          "source_quote": "An ammeter measures current and is connected in series with the component.",
          "proof_verified": true
        },
        {
          "question": "According to Ohm\u2019s law, voltage equals...",
          "options": [
            "current divided by resistance",
            "power divided by current",
            "current multiplied by resistance",
            "resistance multiplied by power"
          ],
          "answer": "current multiplied by resistance",
          "explanation": "The source explicitly states \"The relationship can be written as voltage equals current multiplied by resistance,\" which is Ohm\u2019s law.",
          "source_page": 2,
          "source_quote": "The relationship can be written as voltage equals current multiplied by resistance.",
          "proof_verified": true
        }
      ],
      "full_page_coverage": true,
      "seconds": 9.8
    },
    {
      "document": "electricity",
      "format": "short_answer",
      "requested": 5,
      "started_at": "2026-10-31T05:35:47.273852+00:00",
      "http_status": 200,
      "returned": 5,
      "source_pages": [
        1,
        2,
        3
      ],
      "original_quote_matches": 5,
      "verified_by_service": 5,
      "format_valid": true,
      "answer_positions": [],
      "quiz": [
        {
          "question": "What is the unit used to measure electric charge?",
          "options": [],
          "answer": "coulomb",
          "question_type": "short_answer",
          "explanation": "The source states that electric charge is measured in coulombs, directly naming the unit.",
          "source_page": 1,
          "source_quote": "Electric charge is measured in coulombs.",
          "proof_verified": true
        },
        {
          "question": "According to Ohm\u2019s law, what formula relates voltage, current, and resistance?",
          "options": [],
          "answer": "voltage equals current multiplied by resistance",
          "question_type": "short_answer",
          "explanation": "The source gives the relationship as voltage equals current multiplied by resistance.",
          "source_page": 2,
          "source_quote": "The relationship can be written as voltage equals current multiplied by resistance.",
          "proof_verified": true
        },
        {
          "question": "How is the total resistance of components connected in series determined?",
          "options": [],
          "answer": "It is the sum of the individual resistances.",
          "question_type": "short_answer",
          "explanation": "The passage explains that in series, the total resistance is the sum of each component\u2019s resistance.",
          "source_page": 3,
          "source_quote": "The total resistance is the sum of the individual resistances.",
          "proof_verified": true
        },
        {
          "question": "How much charge does a current of one ampere transfer each second?",
          "options": [],
          "answer": "one coulomb",
          "question_type": "short_answer",
          "explanation": "The passage explains that a current of one ampere transfers one coulomb of charge each second.",
          "source_page": 1,
          "source_quote": "A current of one ampere transfers one coulomb of charge each second.",
          "proof_verified": true
        },
        {
          "question": "How is electrical power calculated from voltage and current?",
          "options": [],
          "answer": "power equals voltage multiplied by current",
          "question_type": "short_answer",
          "explanation": "The source states that power equals voltage multiplied by current.",
          "source_page": 3,
          "source_quote": "Power equals voltage multiplied by current.",
          "proof_verified": true
        }
      ],
      "full_page_coverage": true,
      "seconds": 6.3
    },
    {
      "document": "electricity",
      "format": "true_false",
      "requested": 5,
      "started_at": "2026-10-31T05:35:38.698541+00:00",
      "http_status": 200,
      "returned": 5,
      "source_pages": [
        1,
        2,
        3
      ],
      "original_quote_matches": 5,
      "verified_by_service": 5,
      "format_valid": true,
      "answer_positions": [
        2,
        1,
        2,
        1,
        1
      ],
      "quiz": [
        {
          "question": "Electric charge is measured in coulombs.",
          "options": [
            "False",
            "True"
          ],
          "answer": "True",
          "question_type": "true_false",
          "explanation": "The source states that \"Electric charge is measured in coulombs,\" directly confirming the statement.",
          "source_page": 1,
          "source_quote": "Electric charge is measured in coulombs.",
          "proof_verified": true
        },
        {
          "question": "A voltmeter is connected in parallel across a component to measure potential difference.",
          "options": [
            "True",
            "False"
          ],
          "answer": "True",
          "question_type": "true_false",
          "explanation": "The passage explicitly notes that a voltmeter measures potential difference and is connected in parallel across a component.",
          "source_page": 2,
          "source_quote": "A voltmeter measures potential difference and is connected in parallel across a component.",
          "proof_verified": true
        },
        {
          "question": "In a series circuit, the total resistance is the product of the individual resistances.",
          "options": [
            "True",
            "False"
          ],
          "answer": "False",
          "question_type": "true_false",
          "explanation": "The source states that in series, \"the total resistance is the sum of the individual resistances,\" not the product.",
          "source_page": 3,
          "source_quote": "the total resistance is the sum of the individual resistances.",
          "proof_verified": true
        },
        {
          "question": "An ammeter is connected in parallel with a component to measure current.",
          "options": [
            "False",
            "True"
          ],
          "answer": "False",
          "question_type": "true_false",
          "explanation": "The source says an ammeter is connected in series, not parallel, with the component it measures.",
          "source_page": 1,
          "source_quote": "An ammeter measures current and is connected in series with the component.",
          "proof_verified": true
        },
        {
          "question": "Heating a metal wire always decreases its resistance.",
          "options": [
            "False",
            "True"
          ],
          "answer": "False",
          "question_type": "true_false",
          "explanation": "The source indicates that heating a metal wire usually increases its resistance, contradicting the statement.",
          "source_page": 2,
          "source_quote": "Heating a metal wire usually increases its resistance.",
          "proof_verified": true
        }
      ],
      "full_page_coverage": true,
      "seconds": 8.6
    },
    {
      "document": "water-cycle",
      "format": "mcq",
      "requested": 5,
      "started_at": "2026-10-31T05:35:55.661288+00:00",
      "http_status": 200,
      "returned": 5,
      "source_pages": [
        1,
        2,
        3
      ],
      "original_quote_matches": 5,
      "verified_by_service": 5,
      "format_valid": true,
      "answer_positions": [
        3,
        2,
        4,
        1,
        3
      ],
      "quiz": [
        {
          "question": "Which process changes liquid water into water vapour?",
          "options": [
            "Condensation",
            "Infiltration",
            "Evaporation",
            "Precipitation"
          ],
          "answer": "Evaporation",
          "explanation": "The source quote states that \"Evaporation changes liquid water into water vapour,\" directly defining evaporation as the process that converts liquid water to vapour.",
          "source_page": 1,
          "source_quote": "Evaporation changes liquid water into water vapour.",
          "proof_verified": true
        },
        {
          "question": "How is a drainage basin defined?",
          "options": [
            "volume of water passing a point per unit time",
            "area of land drained by a river and its tributaries",
            "boundary between neighbouring drainage basins",
            "place where two streams or rivers meet"
          ],
          "answer": "area of land drained by a river and its tributaries",
          "explanation": "The source states \"A drainage basin is the area of land drained by a river and its tributaries,\" matching the chosen definition.",
          "source_page": 2,
          "source_quote": "A drainage basin is the area of land drained by a river and its tributaries.",
          "proof_verified": true
        },
        {
          "question": "In a storm hydrograph, what is the term for the highest discharge reached during the event?",
          "options": [
            "Rising limb",
            "Falling limb",
            "Lag time",
            "Peak discharge"
          ],
          "answer": "Peak discharge",
          "explanation": "The quote says \"Peak discharge is the highest discharge reached during the event,\" defining the term precisely.",
          "source_page": 3,
          "source_quote": "Peak discharge is the highest discharge reached during the event.",
          "proof_verified": true
        },
        {
          "question": "What term describes the combined transfer of water to the atmosphere by both evaporation and transpiration?",
          "options": [
            "Evapotranspiration",
            "Percolation",
            "Condensation",
            "Surface runoff"
          ],
          "answer": "Evapotranspiration",
          "explanation": "The quote reads \"Evapotranspiration combines evaporation and transpiration as transfers to the atmosphere,\" confirming that evapotranspiration is the combined process.",
          "source_page": 1,
          "source_quote": "Evapotranspiration combines evaporation and transpiration as transfers to the atmosphere.",
          "proof_verified": true
        },
        {
          "question": "What term refers to the boundary separating neighboring drainage basins?",
          "options": [
            "Tributary",
            "Confluence",
            "Watershed",
            "Infiltration"
          ],
          "answer": "Watershed",
          "explanation": "The text defines \"A watershed is the boundary between neighbouring drainage basins,\" directly linking watershed to the boundary concept.",
          "source_page": 2,
          "source_quote": "A watershed is the boundary between neighbouring drainage basins.",
          "proof_verified": true
        }
      ],
      "full_page_coverage": true,
      "seconds": 8.9
    },
    {
      "document": "water-cycle-scan",
      "format": "mcq",
      "requested": 5,
      "started_at": "2026-10-31T05:36:11.481268+00:00",
      "http_status": 200,
      "returned": 5,
      "source_pages": [
        1,
        2,
        3
      ],
      "original_quote_matches": 5,
      "verified_by_service": 5,
      "format_valid": true,
      "answer_positions": [
        3,
        2,
        1,
        4,
        2
      ],
      "quiz": [
        {
          "question": "What process changes liquid water into water vapour?",
          "options": [
            "Precipitation",
            "Percolation",
            "Evaporation",
            "Condensation"
          ],
          "answer": "Evaporation",
          "explanation": "The source quote states that \"Evaporation changes liquid water into water vapour,\" directly defining evaporation as the process that converts liquid water to vapour.",
          "source_page": 1,
          "source_quote": "Evaporation changes liquid water into water vapour.",
          "proof_verified": true,
          "proof_method": "ocr"
        },
        {
          "question": "What defines the area of land drained by a river and its tributaries?",
          "options": [
            "Watershed",
            "Drainage basin",
            "Confluence",
            "Floodplain"
          ],
          "answer": "Drainage basin",
          "explanation": "The text defines \"A drainage basin is the area of land drained by a river and its tributaries,\" directly answering the question.",
          "source_page": 2,
          "source_quote": "A drainage basin is the area of land drained by a river and its tributaries.",
          "proof_verified": true,
          "proof_method": "ocr"
        },
        {
          "question": "In a storm hydrograph, what is the term for the highest discharge reached during the event?",
          "options": [
            "Peak discharge",
            "Rising limb",
            "Falling limb",
            "Lag time"
          ],
          "answer": "Peak discharge",
          "explanation": "The source notes \"Peak discharge is the highest discharge reached during the event,\" defining the term precisely.",
          "source_page": 3,
          "source_quote": "Peak discharge is the highest discharge reached during the event.",
          "proof_verified": true,
          "proof_method": "ocr"
        },
        {
          "question": "Which term describes the release of water vapour from plants?",
          "options": [
            "Evapotranspiration",
            "Infiltration",
            "Condensation",
            "Transpiration"
          ],
          "answer": "Transpiration",
          "explanation": "The quote reads \"Transpiration is the release of water vapour from plants,\" identifying transpiration as the correct process.",
          "source_page": 1,
          "source_quote": "Transpiration is the release of water vapour from plants.",
          "proof_verified": true,
          "proof_method": "ocr"
        },
        {
          "question": "What is the term for the boundary between neighboring drainage basins?",
          "options": [
            "Tributary",
            "Watershed",
            "Confluence",
            "Infiltration"
          ],
          "answer": "Watershed",
          "explanation": "According to the source, \"A watershed is the boundary between neighbouring drainage basins,\" making watershed the correct term.",
          "source_page": 2,
          "source_quote": "A watershed is the boundary between neighbouring drainage basins.",
          "proof_verified": true,
          "proof_method": "ocr"
        }
      ],
      "full_page_coverage": true,
      "seconds": 20.5
    },
    {
      "document": "water-cycle-scan",
      "format": "short_answer",
      "requested": 5,
      "started_at": "2026-10-31T05:36:44.123599+00:00",
      "http_status": 200,
      "returned": 5,
      "source_pages": [
        1,
        2,
        3
      ],
      "original_quote_matches": 5,
      "verified_by_service": 5,
      "format_valid": true,
      "answer_positions": [],
      "quiz": [
        {
          "question": "What process changes liquid water into water vapour, driven by solar energy?",
          "options": [],
          "answer": "Evaporation",
          "question_type": "short_answer",
          "explanation": "The source states \"Evaporation changes liquid water into water vapour,\" directly defining the process.",
          "source_page": 1,
          "source_quote": "Evaporation changes liquid water into water vapour.",
          "proof_verified": true,
          "proof_method": "ocr"
        },
        {
          "question": "What is the term for the area of land drained by a river and its tributaries?",
          "options": [],
          "answer": "Drainage basin",
          "question_type": "short_answer",
          "explanation": "The source defines it as \"A drainage basin is the area of land drained by a river and its tributaries.\"",
          "source_page": 2,
          "source_quote": "A drainage basin is the area of land drained by a river and its tributaries.",
          "proof_verified": true,
          "proof_method": "ocr"
        },
        {
          "question": "In a storm hydrograph, what is the term for the highest discharge reached during the event?",
          "options": [],
          "answer": "Peak discharge",
          "question_type": "short_answer",
          "explanation": "The text states \"Peak discharge is the highest discharge reached during the event,\" defining the term.",
          "source_page": 3,
          "source_quote": "Peak discharge is the highest discharge reached during the event.",
          "proof_verified": true,
          "proof_method": "ocr"
        },
        {
          "question": "What term describes the combined transfer of water to the atmosphere from both evaporation and plant transpiration?",
          "options": [],
          "answer": "Evapotranspiration",
          "question_type": "short_answer",
          "explanation": "The text explains \"Evapotranspiration combines evaporation and transpiration as transfers to the atmosphere,\" naming the combined process.",
          "source_page": 1,
          "source_quote": "Evapotranspiration combines evaporation and transpiration as transfers to the atmosphere.",
          "proof_verified": true,
          "proof_method": "ocr"
        },
        {
          "question": "What is the boundary called that separates neighboring drainage basins?",
          "options": [],
          "answer": "Watershed",
          "question_type": "short_answer",
          "explanation": "The text states \"A watershed is the boundary between neighbouring drainage basins,\" naming the boundary.",
          "source_page": 2,
          "source_quote": "A watershed is the boundary between neighbouring drainage basins.",
          "proof_verified": true,
          "proof_method": "ocr"
        }
      ],
      "full_page_coverage": true,
      "seconds": 18.1
    },
    {
      "document": "water-cycle-scan",
      "format": "true_false",
      "requested": 5,
      "started_at": "2026-10-31T05:36:31.944029+00:00",
      "http_status": 503,
      "error": "{\"detail\":\"Quiz generation temporarily unavailable\"}",
      "seconds": 12.2
    },
    {
      "document": "water-cycle",
      "format": "short_answer",
      "requested": 5,
      "started_at": "2026-10-31T05:35:58.608779+00:00",
      "http_status": 503,
      "error": "{\"detail\":\"Quiz generation temporarily unavailable\"}",
      "seconds": 2.1
    },
    {
      "document": "water-cycle",
      "format": "true_false",
      "requested": 5,
      "started_at": "2026-10-31T05:35:55.662277+00:00",
      "http_status": 503,
      "error": "{\"detail\":\"Quiz generation temporarily unavailable\"}",
      "seconds": 2.9
    }
  ]
}