A Developer’s Guide to OmniParseBench: Leaderboard Benchmarks and OCR Testing

In this tutorial, we implement OmniParseBench , Datalab’s open benchmark for document parsers, which grades a parser’s output with 16,288 small pass-or-fail tests across 2,937 pages, from scanned forms and handwriting to arXiv math and dense tables. Rather than trust the leaderboard, we rebuild it from the verdicts Datalab released for 27 parsers, after feeding the scorer a hand-made page to see which mistakes flip a verdict. We then measure ranking certainty, slice it by document type, test whether the two models the card names as helping build tests behave differently on tests a model corrected, and run Tesseract and Indra OCR ourselves on a 50-page sample. Along the way, we find that the top two parsers are statistically tied, the leader changes on most slices, and one launch figure doesn’t match the released files. Copy Code Copied Use a different Browser import os import re import sys import json import time import html import shutil import platform import traceback import subprocess import collections import urllib.request from pathlib import Path RESULTS = {} # step -> one-line result, for the summary table CLAIMS = [] # (step, claim, held) CURRENT = ["1"] def banner(title): print("\n" + "=" * 78) print(title) print("=" * 78) def section(name): def wrap(fn): def run(*a, **kw): CURRENT[0] = name.split(".")[0] banner(name) t0 = time.time() try: out = fn(*a, **kw) RESULTS[name] = out if isinstance(out, str) else "ok" return out except Exception as e: RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}" print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}") traceback.print_exc(limit=3) return None finally: print(f"\n[{time.time() - t0:.1f} s]") return run return wrap def claim(text, held): CLAIMS.append((CURRENT[0], text, bool(held))) return held banner("1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision") SYSTEM = platform.system() OPB_VERSION = "1.0.0rc1" # a pre-release: a bare `pip install omni-parse-bench` resolves to 0.2.0, whose API differs REPO = "datalab-to/omni_parse_bench" REV = "961c0f89a2da701f5ce53afbd5e80aa469b0369a" # the dataset with the published verdicts of 27 parsers INDRA = ["indra-ocr==0.3.2", "numpy==2.5.4", "onnxruntime==1.31.0", "opencv-python-headless==5.0.0.93", "pypdfium2==5.14.0", "huggingface_hub==2.2.0"] # the pins of the package's own Indra server image WORK = Path(os.environ.get("OPB_WORK") or ("/content/opb" if os.path.isdir("/content") else "opb_work")).resolve() WORK.mkdir(parents=True, exist_ok=True) subprocess.run([sys.executable, "-m", "pip", "install", "-q", f"omni-parse-bench[benchmark]=={OPB_VERSION}", "uv"], check=True) if SYSTEM == "Linux" and shutil.which("tesseract") is None: scripts = ("arab armn beng cans cher cyrl deva ethi frak geor grek gujr guru hang hang-vert hans hans-vert hant " "hant-vert hebr jpan jpan-vert knda khmr laoo latn mlym mymr orya sinh syrc taml telu thaa thai tibt viet").split() subprocess.run(["apt-get", "-qq", "update"], check=True) subprocess.run(["apt-get", "-qq", "install", "-y", "tesseract-ocr", "tesseract-ocr-eng", "tesseract-ocr-osd"] + [f"tesseract-ocr-script-{s}" for s in scripts], check=True, stdout=subprocess.DEVNULL) import numpy as np import pandas as pd import importlib.metadata as md from huggingface_hub import snapshot_download from omni_parse_bench import benchmark, data, harness, markdown, vocab from omni_parse_bench.scorers import unit_tests print(f"omni-parse-bench {md.version('omni-parse-bench')}, Python {platform.python_version()}, {SYSTEM} {platform.machine()}, " f"{os.cpu_count()} CPU cores") if SYSTEM == "Linux": print("os:", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0]) tess = subprocess.run(["tesseract", "--version"], capture_output=True, text=True) print("tesseract:", (tess.stdout or tess.stderr).splitlines()[0]) TESS_MAJOR = int((tess.stdout or tess.stderr).split()[1].split(".")[0]) print("test families:", {f: [t for t, g in vocab.FAMILIES.items() if g == f] for f in dict.fromkeys(vocab.FAMILIES.values())}) print(f"dataset {REPO} @ {REV[:7]}; work folder {WORK}") RESULTS["1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision"] = ( f"omni-parse-bench {md.version('omni-parse-bench')}, tesseract {(tess.stdout or tess.stderr).split()[1]}, dataset @ {REV[:7]}") We install omni-parse-bench pinned to 1.0.0rc1, because a bare pip install still resolves to 0.2.0, an older release with a different API. The benchmark extra brings the parser adapters, the Hugging Face client, and pyarrow for the manifest, and on Colab we add Tesseract with its script models through apt. We also pin the dataset to one commit, since the repository changes as parsers are added. Printing the families shows the structure the benchmark rests on: present, order, and repeat tests form the text family; table_cell forms the tables family; and layout_kind forms the layout family. Copy Code Copied Use a different Browser @section("2. How the scorer decides: one hand-built page, five test types, one change at a time") def scorer_anatomy(): T = lambda i, tt, out, tags, args, st=None: {"id": i, "test_type": tt, "output_type": out, "derivation": "authored", "tags": tags, "args": args, "settings": st or {}} exact = lambda s: {"text": s, "max_diffs": 0} gold = [ T("title", "present", "html", ["heading"], {"content": exact("Quarterly Report")}), T("total", "present", "html", ["eng"], {"content": exact("Total due")}), T("growth", "present", "html", ["eng"], {"content": {"number": "-4.8%"}}), T("amount", "present", "html", ["eng"], {"content": {"number": "1,240.00"}}), T("eq_ratio", "present", "html", ["math"], {"content": {"latex": r"\alpha_{1} + \beta"}}), T("eq_half", "present", "html", ["math"], {"content": {"latex": r"\frac{1}{2} \to x"}}), T("order", "order", "html", ["eng"], {"before": exact("Introduction: this report covers the second quarter."), "after": exact("Conclusion: the plan holds.")}), T("repeat", "repeat", "html", ["eng"], {"content": exact("Revenue")}, {"expected_count": 2}), T("cell_top", "table_cell", "html", ["table"], {"cell": exact("1,240"), "top_heading": exact("2024")}), T("cell_left", "table_cell", "html", ["table"], {"cell": exact("1,240"), "left_heading": exact("Income")}), T("cell_price", "table_cell", "html", ["table"], {"cell": exact("9.50"), "top_heading": exact("Price"), "left": exact("Widget")}), T("block_title", "layout_kind", "blocks", ["heading"], {"content": dict(exact("Quarterly Report"), place=[[0.10, 0.04, 0.90, 0.09]])}, {"kind": "heading"}), T("block_total", "layout_kind", "blocks", ["eng"], {"content": dict(exact("Total due"), place=[[0.10, 0.12, 0.90, 0.16]])}, {"kind": "text"}), ] gold_file = WORK / "fixture" / "gold" / "demo-p1.json" # a gold file is named after its page gold_file.parent.mkdir(parents=True, exist_ok=True) gold_file.write_text(json.dumps({"tests": gold}, indent=1)) page = data.Page(page_id="demo-p1", document="demo", suite="demo", doc_path=WORK / "fixture" / "demo-p1.pdf", gt_path=gold_file, outputs=("blocks", "html")) tests = unit_tests.gold(page) table_a = ("20232024Income1,0001,240" "Costs600700") table_b = "ProductPriceWidget9.50" md_table_b = "| Product | Price |\n|---|---|\n| Widget | 9.50 |" parts = {"title": "Quarterly Report", "total": "

Total due: 1,240.00 USD.

", "growth": "

Growth was -4.8% year on year.

", "ratio": r"

Ratio: 1+ holds.

", "half": r"

Half: 12x in the model.

", "intro": "

Introduction: this report covers the second quarter.

", "revenue": "

Revenue is shown in thousands. Revenue grew in the second quarter.

", "table_a": table_a, "conclusion": "

Conclusion: the plan holds.

", "table_b": table_b} blocks = [{"label": "heading", "bbox": [0.10, 0.04, 0.90, 0.09], "text": "Quarterly