{
  "benchmark": "VoiceBrief PDF-to-audio extraction accessibility benchmark",
  "version": "2026-07-28",
  "evaluatedAt": "2026-07-30T08:52:10.851Z",
  "sourceRevision": "c343c1cbfdb5",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "environment": {
    "runner": "darwin",
    "node": "v25.8.1",
    "standardExtractor": "pdf-parse 1.1.1 through VoiceBrief server/pdfProcessor.ts",
    "ocrFallbackRequested": true,
    "ocrFallbackAvailable": true,
    "ocrModelWhenUsed": "gpt-4o",
    "note": "The runner used synthetic VoiceBrief-owned PDFs. When --vision is enabled, current environment variables configure the same OCR code path as PDF uploads; this is not a production web request."
  },
  "scoring": {
    "passRule": "A case passes only when every expected anchor is recovered, every adjacent anchor pair appears in the intended order, and repeated-furniture limits are respected.",
    "casesPassed": 7,
    "totalCases": 7,
    "passRatePercent": 100
  },
  "limitations": [
    "This is an extraction benchmark, not a WCAG, PDF/UA, ADA, or Section 508 conformance audit.",
    "The corpus is deliberately small and synthetic; it does not estimate performance across all real-world PDFs.",
    "It does not score voice naturalness, pronunciation, equations, charts, assistive-technology navigation, or audio export.",
    "OCR output can vary when the external model or image-processing stack changes."
  ],
  "cases": [
    {
      "id": "single-column",
      "name": "Single-column born-digital PDF",
      "challenge": "Baseline text recovery and reading order",
      "extractionMethod": "standard",
      "pages": 1,
      "extractedWordCount": 19,
      "recoveredAnchors": 3,
      "totalAnchors": 3,
      "correctOrderPairs": 2,
      "totalOrderPairs": 2,
      "noiseViolations": [],
      "passed": true,
      "expectedReadingOrder": [
        "ALPHA ORCHARD",
        "BRAVO RIVER",
        "CHARLIE LANTERN"
      ],
      "extractedText": "PDF Accessibility Benchmark\n\nALPHA ORCHARD begins the first paragraph.\n\nBRAVO RIVER continues the explanation.\n\nCHARLIE LANTERN closes the page."
    },
    {
      "id": "two-column",
      "name": "Two-column PDF with logical content order",
      "challenge": "Finish the left column before reading the right column",
      "extractionMethod": "standard",
      "pages": 1,
      "extractedWordCount": 24,
      "recoveredAnchors": 6,
      "totalAnchors": 6,
      "correctOrderPairs": 5,
      "totalOrderPairs": 5,
      "noiseViolations": [],
      "passed": true,
      "expectedReadingOrder": [
        "DELTA MAPLE",
        "ECHO TUNNEL",
        "FOXTROT COMPASS",
        "GOLF HARBOR",
        "HOTEL MEADOW",
        "INDIA SUMMIT"
      ],
      "extractedText": "DELTA MAPLE left one.\n\nECHO TUNNEL left two.\n\nFOXTROT COMPASS left three.\n\nGOLF HARBOR right one.\n\nHOTEL MEADOW right two.\n\nINDIA SUMMIT right three."
    },
    {
      "id": "two-column-broken-order",
      "name": "Two-column PDF with broken content order",
      "challenge": "Detect a visually plausible page whose content stream interleaves both columns",
      "extractionMethod": "standard",
      "pages": 1,
      "extractedWordCount": 24,
      "recoveredAnchors": 6,
      "totalAnchors": 6,
      "correctOrderPairs": 5,
      "totalOrderPairs": 5,
      "noiseViolations": [],
      "passed": true,
      "expectedReadingOrder": [
        "JULIET PEBBLE",
        "KILO CANYON",
        "LIMA VIOLET",
        "MIKE FOREST",
        "NOVEMBER BASIN",
        "OSCAR SUNRISE"
      ],
      "extractedText": "JULIET PEBBLE left one.\n\nKILO CANYON left two.\n\nLIMA VIOLET left three.\n\nMIKE FOREST right one.\n\nNOVEMBER BASIN right two.\n\nOSCAR SUNRISE right three."
    },
    {
      "id": "repeated-furniture",
      "name": "Three-page PDF with repeated furniture",
      "challenge": "Avoid narrating the same running header and footer on every page",
      "extractionMethod": "standard",
      "pages": 3,
      "extractedWordCount": 27,
      "recoveredAnchors": 3,
      "totalAnchors": 3,
      "correctOrderPairs": 2,
      "totalOrderPairs": 2,
      "noiseViolations": [],
      "passed": true,
      "expectedReadingOrder": [
        "PAPA CEDAR",
        "QUEBEC VALLEY",
        "ROMEO MARBLE"
      ],
      "extractedText": "PAPA CEDAR is the body text for page 1.\n\nQUEBEC VALLEY is the body text for page 2.\n\nROMEO MARBLE is the body text for page 3."
    },
    {
      "id": "image-only-scan",
      "name": "Image-only scanned PDF",
      "challenge": "Recover text through the same OCR fallback used for textless uploads",
      "extractionMethod": "vision",
      "pages": 1,
      "extractedWordCount": 21,
      "recoveredAnchors": 3,
      "totalAnchors": 3,
      "correctOrderPairs": 2,
      "totalOrderPairs": 2,
      "noiseViolations": [],
      "passed": true,
      "expectedReadingOrder": [
        "SIERRA PLANET",
        "TANGO GARDEN",
        "UNIFORM BRIDGE"
      ],
      "extractedText": "PDF Accessibility Benchmark\n\nSIERRA PLANET begins the scanned page.\n\nTANGO GARDEN continues the scanned page.\n\nUNIFORM BRIDGE ends the scanned page."
    },
    {
      "id": "simple-table",
      "name": "Simple table-like PDF",
      "challenge": "Preserve row-major cell order when table semantics are absent",
      "extractionMethod": "standard",
      "pages": 1,
      "extractedWordCount": 8,
      "recoveredAnchors": 4,
      "totalAnchors": 4,
      "correctOrderPairs": 3,
      "totalOrderPairs": 3,
      "noiseViolations": [],
      "passed": true,
      "expectedReadingOrder": [
        "VICTOR 12",
        "WHISKEY 24",
        "XRAY 36",
        "YANKEE 48"
      ],
      "extractedText": "VICTOR 12\n\nWHISKEY 24\n\nXRAY 36\n\nYANKEE 48"
    },
    {
      "id": "line-break-hyphenation",
      "name": "Line-break hyphenation",
      "challenge": "Rejoin a word split by a visual line break without narrating the hyphen",
      "extractionMethod": "standard",
      "pages": 1,
      "extractedWordCount": 8,
      "recoveredAnchors": 2,
      "totalAnchors": 2,
      "correctOrderPairs": 1,
      "totalOrderPairs": 1,
      "noiseViolations": [],
      "passed": true,
      "expectedReadingOrder": [
        "ZULU MICROSCOPIC",
        "AMBER TELESCOPE"
      ],
      "extractedText": "ZULU MICROSCOPIC evidence begins here.\n\nAMBER TELESCOPE follows."
    }
  ]
}