{
 "openapi": "3.1.0",
 "info": {
  "title": "PuffinParse benchmark data",
  "version": "1.0.0",
  "summary": "Read-only JSON for the open PuffinParse benchmark: leaderboard, runs, per-document scores, model outputs and datasets.",
  "description": "Static files published with the results viewer: plain GET, no authentication, no rate limit beyond the CDN, CORS open. Start at `index.json`; the newest run on the newest `combined-v*` dataset is the headline leaderboard. The data changes only when a new benchmark run is committed to the repository. Result JSON is MIT licensed; every dataset document carries its own licence (see the manifests).",
  "contact": {
   "name": "PuffinParse",
   "url": "https://github.com/ajinkyashejul/puffinparse/issues"
  }
 },
 "externalDocs": {
  "description": "Data API guide with curl examples",
  "url": "https://puffinparse.com/docs/benchmark/data-api/"
 },
 "servers": [
  {
   "url": "https://puffinparse.com"
  }
 ],
 "tags": [
  {
   "name": "Runs",
   "description": "The leaderboard and every scored run."
  },
  {
   "name": "Model outputs",
   "description": "What each model returned for each document."
  },
  {
   "name": "Datasets",
   "description": "Inputs, ground truth and licences."
  }
 ],
 "paths": {
  "/benchmark-results/data/index.json": {
   "get": {
    "operationId": "getIndex",
    "tags": [
     "Runs"
    ],
    "summary": "Leaderboard: every run with per-model summaries",
    "description": "Each run's `models` are sorted by `summary.headline`, best first. Prefer the newest run whose dataset is `combined-v*`.",
    "responses": {
     "200": {
      "description": "The run index.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Index"
        }
       }
      }
     },
     "404": {
      "description": "No such file."
     }
    }
   }
  },
  "/benchmark-results/data/runs/{run_id}.json": {
   "get": {
    "operationId": "getRun",
    "tags": [
     "Runs"
    ],
    "summary": "One full run, with per-document scores",
    "description": "`models[].docs[]` holds every document's metrics, latency and cost.",
    "parameters": [
     {
      "name": "run_id",
      "in": "path",
      "required": true,
      "description": "A `run_id` from `index.json` `runs[]`.",
      "schema": {
       "type": "string"
      },
      "example": "run-20260925T090851Z"
     }
    ],
    "responses": {
     "200": {
      "description": "The full result file.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Run"
        }
       }
      }
     },
     "404": {
      "description": "No such file."
     }
    }
   }
  },
  "/benchmark-results/data/outputs/{run_id}/{model_slug}/{doc_id}.md": {
   "get": {
    "operationId": "getOutputMarkdown",
    "tags": [
     "Model outputs"
    ],
    "summary": "The markdown a model produced for a document (what was scored)",
    "description": "Absent for documents listed in `outputs[slug].missing` (failed calls and research-only sources that are never redistributed).",
    "parameters": [
     {
      "name": "run_id",
      "in": "path",
      "required": true,
      "description": "A `run_id` from `index.json` `runs[]`.",
      "schema": {
       "type": "string"
      },
      "example": "run-20260925T090851Z"
     },
     {
      "name": "model_slug",
      "in": "path",
      "required": true,
      "description": "A model slug (`reducto_r-1`): `runs[].models[].slug`, i.e. the model string with `/` replaced by `_`.",
      "schema": {
       "type": "string"
      },
      "example": "llamaparse_cost_effective"
     },
     {
      "name": "doc_id",
      "in": "path",
      "required": true,
      "description": "A document id from the run. Ids of combined datasets contain one literal `/` (`dpbench/01030000000016`): send it unescaped, it is a directory on the server.",
      "schema": {
       "type": "string"
      },
      "example": "dpbench/01030000000016"
     }
    ],
    "responses": {
     "200": {
      "description": "Markdown.",
      "content": {
       "text/markdown": {
        "schema": {
         "type": "string"
        }
       }
      }
     },
     "404": {
      "description": "No saved output."
     }
    }
   }
  },
  "/benchmark-results/data/outputs/{run_id}/{model_slug}/{doc_id}.json": {
   "get": {
    "operationId": "getOutputResponse",
    "tags": [
     "Model outputs"
    ],
    "summary": "The unified ParseResponse (blocks and boxes) for a document",
    "description": "Only for document ids listed in `outputs[slug].json`.",
    "parameters": [
     {
      "name": "run_id",
      "in": "path",
      "required": true,
      "description": "A `run_id` from `index.json` `runs[]`.",
      "schema": {
       "type": "string"
      },
      "example": "run-20260925T090851Z"
     },
     {
      "name": "model_slug",
      "in": "path",
      "required": true,
      "description": "A model slug (`reducto_r-1`): `runs[].models[].slug`, i.e. the model string with `/` replaced by `_`.",
      "schema": {
       "type": "string"
      },
      "example": "llamaparse_cost_effective"
     },
     {
      "name": "doc_id",
      "in": "path",
      "required": true,
      "description": "A document id from the run. Ids of combined datasets contain one literal `/` (`dpbench/01030000000016`): send it unescaped, it is a directory on the server.",
      "schema": {
       "type": "string"
      },
      "example": "dpbench/01030000000016"
     }
    ],
    "responses": {
     "200": {
      "description": "The unified response.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/ParseResponse"
        }
       }
      }
     },
     "404": {
      "description": "No such file."
     }
    }
   }
  },
  "/benchmark-results/data/datasets/{dataset}/manifest.json": {
   "get": {
    "operationId": "getManifest",
    "tags": [
     "Datasets"
    ],
    "summary": "A dataset manifest: documents, categories, licences",
    "description": "`file`, `truth`, `rules` and `preview(s)` are relative to this manifest's URL (combined datasets point at `../<source>/...`). Documents from research-only sources are listed but their files are not published.",
    "parameters": [
     {
      "name": "dataset",
      "in": "path",
      "required": true,
      "description": "A dataset name: a key of `index.json` `datasets`.",
      "schema": {
       "type": "string"
      },
      "example": "combined-v3"
     }
    ],
    "responses": {
     "200": {
      "description": "The manifest.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Manifest"
        }
       }
      }
     },
     "404": {
      "description": "No such file."
     }
    }
   }
  }
 },
 "components": {
  "schemas": {
   "Index": {
    "type": "object",
    "description": "Every run, every dataset and the labels the viewer shows. The leaderboard of a run is `runs[i].models`, already sorted by `summary.headline`, best first.",
    "required": [
     "generated_at",
     "repo",
     "base",
     "datasets",
     "labels",
     "runs"
    ],
    "properties": {
     "generated_at": {
      "type": "string",
      "format": "date-time"
     },
     "repo": {
      "type": "string",
      "format": "uri"
     },
     "base": {
      "type": "string",
      "description": "URL prefix the viewer was built for."
     },
     "datasets": {
      "type": "object",
      "description": "Dataset name -> summary.",
      "additionalProperties": {
       "$ref": "#/components/schemas/DatasetSummary"
      }
     },
     "labels": {
      "$ref": "#/components/schemas/Labels"
     },
     "runs": {
      "type": "array",
      "description": "Newest first (`created_at`, then `run_id`).",
      "items": {
       "$ref": "#/components/schemas/RunIndexEntry"
      }
     }
    }
   },
   "DatasetSummary": {
    "type": "object",
    "required": [
     "name",
     "documents",
     "categories"
    ],
    "properties": {
     "name": {
      "type": "string"
     },
     "version": {
      "type": [
       "string",
       "null"
      ]
     },
     "description": {
      "type": [
       "string",
       "null"
      ]
     },
     "license": {
      "type": [
       "string",
       "null"
      ],
      "description": "SPDX id, or `mixed` for unions."
     },
     "generator": {
      "type": [
       "string",
       "null"
      ]
     },
     "sources": {
      "type": [
       "array",
       "null"
      ],
      "description": "The source datasets of a combined dataset.",
      "items": {
       "$ref": "#/components/schemas/DatasetSource"
      }
     },
     "attribution": {
      "type": [
       "string",
       "null"
      ]
     },
     "documents": {
      "type": "integer"
     },
     "kinds": {
      "type": "object",
      "description": "Document kind (`transcript`, `rules`) -> count.",
      "additionalProperties": {
       "type": "integer"
      }
     },
     "categories": {
      "type": "array",
      "items": {
       "type": "string"
      }
     },
     "missing": {
      "type": "boolean",
      "description": "Present (true) when the manifest was not found."
     }
    }
   },
   "DatasetSource": {
    "type": "object",
    "required": [
     "name",
     "version",
     "license",
     "path",
     "documents",
     "manifest_sha256"
    ],
    "properties": {
     "name": {
      "type": "string"
     },
     "version": {
      "type": "string"
     },
     "license": {
      "type": "string"
     },
     "path": {
      "type": "string",
      "description": "Relative to the combined dataset's manifest."
     },
     "documents": {
      "type": "integer"
     },
     "manifest_sha256": {
      "type": "string"
     },
     "attribution": {
      "type": "string"
     }
    }
   },
   "Labels": {
    "type": "object",
    "required": [
     "sources",
     "categories"
    ],
    "properties": {
     "sources": {
      "type": "object",
      "description": "Source id (`olmocr`) -> display name (`olmOCR-bench`).",
      "additionalProperties": {
       "type": "string"
      }
     },
     "categories": {
      "type": "object",
      "description": "Category id (`table_tests`) -> display name (`Tables`).",
      "additionalProperties": {
       "type": "string"
      }
     }
    }
   },
   "RunDataset": {
    "type": "object",
    "required": [
     "name",
     "version",
     "documents",
     "sha256"
    ],
    "properties": {
     "name": {
      "type": "string"
     },
     "version": {
      "type": "string"
     },
     "documents": {
      "type": "integer"
     },
     "sha256": {
      "type": "string",
      "description": "Covers the manifest and every input, truth and rule file."
     }
    }
   },
   "Normalize": {
    "type": "object",
    "properties": {
     "case_insensitive": {
      "type": "boolean"
     },
     "strip_markdown": {
      "type": "boolean"
     },
     "strip_punctuation": {
      "type": "boolean"
     }
    }
   },
   "RunIndexEntry": {
    "type": "object",
    "description": "A run without its per-document records (those are in `file`).",
    "required": [
     "run_id",
     "dataset",
     "categories",
     "models",
     "outputs",
     "file"
    ],
    "properties": {
     "run_id": {
      "type": "string"
     },
     "created_at": {
      "type": [
       "string",
       "null"
      ],
      "format": "date-time"
     },
     "puffinparse_version": {
      "type": [
       "string",
       "null"
      ]
     },
     "scorer_version": {
      "type": [
       "integer",
       "null"
      ],
      "description": "Absent/null means scorer v1."
     },
     "dataset": {
      "$ref": "#/components/schemas/RunDataset"
     },
     "normalize": {
      "$ref": "#/components/schemas/Normalize"
     },
     "categories": {
      "type": "array",
      "items": {
       "type": "string"
      }
     },
     "models": {
      "type": "array",
      "description": "Sorted by `summary.headline`, best first: this is the leaderboard.",
      "items": {
       "$ref": "#/components/schemas/RankedModel"
      }
     },
     "outputs": {
      "type": "object",
      "description": "Model slug -> which saved outputs exist.",
      "additionalProperties": {
       "$ref": "#/components/schemas/OutputInventory"
      }
     },
     "file": {
      "type": "string",
      "description": "The full run, relative to `/benchmark-results/`."
     }
    }
   },
   "RankedModel": {
    "type": "object",
    "required": [
     "model",
     "slug",
     "summary"
    ],
    "properties": {
     "model": {
      "type": "string",
      "description": "`<provider>/<model>`, e.g. `reducto/r-1`."
     },
     "slug": {
      "type": "string",
      "description": "`model` with `/` replaced by `_` (output directory name)."
     },
     "summary": {
      "$ref": "#/components/schemas/ModelSummary"
     }
    }
   },
   "OutputInventory": {
    "type": "object",
    "required": [
     "json",
     "missing"
    ],
    "properties": {
     "json": {
      "type": "array",
      "items": {
       "type": "string"
      },
      "description": "Document ids that also have a `<doc>.json` unified response."
     },
     "missing": {
      "type": "array",
      "items": {
       "type": "string"
      },
      "description": "Document ids with no published `<doc>.md`."
     }
    }
   },
   "ModelSummary": {
    "type": "object",
    "required": [
     "documents",
     "failed",
     "overall"
    ],
    "properties": {
     "documents": {
      "type": "integer"
     },
     "failed": {
      "type": "integer"
     },
     "headline": {
      "type": "number",
      "description": "The number models are ranked by (0..1)."
     },
     "char_similarity": {
      "type": "number",
      "description": "0..1, higher is better."
     },
     "cer": {
      "type": "number",
      "description": "Character error rate."
     },
     "wer": {
      "type": "number",
      "description": "Word error rate."
     },
     "word_f1": {
      "type": "number",
      "description": "0..1, higher is better."
     },
     "order_score": {
      "type": "number",
      "description": "Reading-order agreement."
     },
     "table_score": {
      "type": "number",
      "description": "Table similarity (documents with tables only)."
     },
     "teds_grid": {
      "type": "number",
      "description": "Tree-edit-distance similarity of table grids."
     },
     "rule_pass_rate": {
      "type": "number",
      "description": "Share of assertions passed (rules documents only)."
     },
     "overall": {
      "type": "number",
      "description": "`headline` x 100."
     },
     "latency_p50_ms": {
      "type": "integer"
     },
     "latency_p95_ms": {
      "type": "integer"
     },
     "latency_per_page_ms": {
      "type": "number"
     },
     "total_pages": {
      "type": "integer"
     },
     "total_cost_usd": {
      "type": "number"
     },
     "cost_per_1k_pages_usd": {
      "type": "number"
     },
     "empty_outputs": {
      "type": "integer"
     },
     "by_category": {
      "type": "object",
      "description": "Category id -> the same metrics over that category.",
      "additionalProperties": {
       "$ref": "#/components/schemas/CategorySummary"
      }
     }
    }
   },
   "CategorySummary": {
    "type": "object",
    "required": [
     "documents",
     "failed",
     "overall"
    ],
    "properties": {
     "documents": {
      "type": "integer"
     },
     "failed": {
      "type": "integer"
     },
     "headline": {
      "type": "number",
      "description": "The number models are ranked by (0..1)."
     },
     "char_similarity": {
      "type": "number",
      "description": "0..1, higher is better."
     },
     "cer": {
      "type": "number",
      "description": "Character error rate."
     },
     "wer": {
      "type": "number",
      "description": "Word error rate."
     },
     "word_f1": {
      "type": "number",
      "description": "0..1, higher is better."
     },
     "order_score": {
      "type": "number",
      "description": "Reading-order agreement."
     },
     "table_score": {
      "type": "number",
      "description": "Table similarity (documents with tables only)."
     },
     "teds_grid": {
      "type": "number",
      "description": "Tree-edit-distance similarity of table grids."
     },
     "rule_pass_rate": {
      "type": "number",
      "description": "Share of assertions passed (rules documents only)."
     },
     "overall": {
      "type": "number",
      "description": "`headline` x 100."
     }
    }
   },
   "Run": {
    "type": "object",
    "description": "One benchmark result file as committed under `benchmark/results/` (docs/SPEC.md section 10.4), with every per-document score.",
    "required": [
     "run_id",
     "dataset",
     "models"
    ],
    "properties": {
     "run_id": {
      "type": "string"
     },
     "created_at": {
      "type": "string",
      "format": "date-time"
     },
     "puffinparse_version": {
      "type": "string"
     },
     "scorer_version": {
      "type": "integer"
     },
     "rescored_at": {
      "type": "string",
      "format": "date-time"
     },
     "rescore_kept_docs": {
      "type": "integer"
     },
     "dataset": {
      "$ref": "#/components/schemas/RunDataset"
     },
     "normalize": {
      "$ref": "#/components/schemas/Normalize"
     },
     "models": {
      "type": "array",
      "items": {
       "$ref": "#/components/schemas/RunModel"
      }
     }
    }
   },
   "RunModel": {
    "type": "object",
    "required": [
     "model",
     "docs",
     "summary"
    ],
    "properties": {
     "model": {
      "type": "string"
     },
     "summary": {
      "$ref": "#/components/schemas/ModelSummary"
     },
     "docs": {
      "type": "array",
      "items": {
       "$ref": "#/components/schemas/DocumentScore"
      }
     }
    }
   },
   "DocumentScore": {
    "type": "object",
    "description": "One model on one document. A failed call has `error` (and `error_kind`) instead of `metrics` / `headline` / `cost_usd`.",
    "required": [
     "id",
     "category",
     "kind",
     "table_only",
     "pages",
     "latency_ms"
    ],
    "properties": {
     "id": {
      "type": "string",
      "description": "Document id; may contain `/` (`dpbench/0103...`)."
     },
     "category": {
      "type": "string"
     },
     "kind": {
      "type": "string",
      "description": "`transcript` (scored against truth) or `rules`."
     },
     "table_only": {
      "type": "boolean"
     },
     "pages": {
      "type": "integer"
     },
     "metrics": {
      "$ref": "#/components/schemas/DocumentMetrics"
     },
     "headline": {
      "type": "number",
      "description": "0..1, higher is better."
     },
     "latency_ms": {
      "type": "integer"
     },
     "cost_usd": {
      "type": "number"
     },
     "empty_output": {
      "type": "boolean"
     },
     "error": {
      "type": "string"
     },
     "error_kind": {
      "type": "string"
     },
     "provider_job_id": {
      "type": "string"
     },
     "cache_hit": {
      "type": [
       "boolean",
       "null"
      ]
     },
     "attempts": {
      "type": "integer"
     },
     "started_at": {
      "type": "string",
      "format": "date-time"
     }
    }
   },
   "DocumentMetrics": {
    "type": "object",
    "required": [
     "char_similarity",
     "cer",
     "wer",
     "word_f1"
    ],
    "properties": {
     "char_similarity": {
      "type": "number",
      "description": "0..1, higher is better."
     },
     "cer": {
      "type": "number",
      "description": "Character error rate."
     },
     "wer": {
      "type": "number",
      "description": "Word error rate."
     },
     "word_f1": {
      "type": "number",
      "description": "0..1, higher is better."
     },
     "order_score": {
      "type": "number",
      "description": "Reading-order agreement."
     },
     "table_score": {
      "type": "number",
      "description": "Table similarity (documents with tables only)."
     },
     "teds_grid": {
      "type": "number",
      "description": "Tree-edit-distance similarity of table grids."
     },
     "rule_pass_rate": {
      "type": "number",
      "description": "Share of assertions passed (rules documents only)."
     },
     "word_recall": {
      "type": "number",
      "description": "0..1, higher is better."
     },
     "word_precision": {
      "type": "number",
      "description": "0..1, higher is better."
     },
     "pred_chars": {
      "type": "integer"
     },
     "truth_chars": {
      "type": "integer"
     },
     "rules_passed": {
      "type": "integer"
     },
     "rules_total": {
      "type": "integer"
     }
    }
   },
   "Manifest": {
    "type": "object",
    "description": "A dataset manifest, plus the display names the viewer build adds (`title`, `ordinal`, `source_label`, `category_label`) and page previews.",
    "required": [
     "name",
     "version",
     "documents"
    ],
    "properties": {
     "name": {
      "type": "string"
     },
     "version": {
      "type": "string"
     },
     "description": {
      "type": "string"
     },
     "license": {
      "type": "string"
     },
     "generator": {
      "type": "string"
     },
     "sources": {
      "type": "array",
      "items": {
       "$ref": "#/components/schemas/DatasetSource"
      }
     },
     "notes": {
      "type": "array",
      "items": {
       "type": "string"
      }
     },
     "documents": {
      "type": "array",
      "items": {
       "$ref": "#/components/schemas/ManifestDocument"
      }
     }
    }
   },
   "ManifestDocument": {
    "type": "object",
    "description": "Every path is relative to the manifest's own URL.",
    "required": [
     "id",
     "file",
     "truth",
     "pages",
     "category",
     "tags",
     "title"
    ],
    "properties": {
     "id": {
      "type": "string"
     },
     "file": {
      "type": "string",
      "description": "The input document."
     },
     "truth": {
      "type": "string",
      "description": "Ground-truth markdown; empty for `rules` documents."
     },
     "rules": {
      "type": "string",
      "description": "Machine-checkable assertions (`kind: rules`)."
     },
     "pages": {
      "type": "integer"
     },
     "category": {
      "type": "string"
     },
     "tags": {
      "type": "array",
      "items": {
       "type": "string"
      }
     },
     "kind": {
      "type": "string"
     },
     "source_id": {
      "type": "string"
     },
     "sha256": {
      "type": "string"
     },
     "license": {
      "type": "string"
     },
     "attribution": {
      "type": "string"
     },
     "preview": {
      "type": "string",
      "description": "Page-1 image."
     },
     "previews": {
      "type": "array",
      "items": {
       "type": "string"
      }
     },
     "title": {
      "type": "string"
     },
     "ordinal": {
      "type": "integer"
     },
     "source_label": {
      "type": "string"
     },
     "category_label": {
      "type": "string"
     }
    }
   },
   "ParseResponse": {
    "type": "object",
    "description": "The unified PuffinParse parse response the model returned (docs/SPEC.md section 4). Blocks carry normalised (0..1) bounding boxes.",
    "required": [
     "id",
     "provider",
     "model",
     "pages",
     "markdown",
     "usage"
    ],
    "properties": {
     "id": {
      "type": "string"
     },
     "provider": {
      "type": "string"
     },
     "model": {
      "type": "string"
     },
     "provider_job_id": {
      "type": "string"
     },
     "markdown": {
      "type": "string"
     },
     "text": {
      "type": "string"
     },
     "pages": {
      "type": "array",
      "items": {
       "$ref": "#/components/schemas/Page"
      }
     },
     "usage": {
      "type": "object",
      "required": [
       "pages"
      ],
      "properties": {
       "pages": {
        "type": "integer"
       },
       "credits": {
        "type": "number"
       }
      }
     },
     "cost_usd": {
      "type": [
       "number",
       "null"
      ]
     },
     "latency_ms": {
      "type": "integer"
     },
     "created_at": {
      "type": "string",
      "format": "date-time"
     },
     "metadata": {
      "type": "object",
      "description": "Provider-specific extras."
     }
    }
   },
   "Page": {
    "type": "object",
    "required": [
     "page_number",
     "blocks"
    ],
    "properties": {
     "page_number": {
      "type": "integer"
     },
     "markdown": {
      "type": "string"
     },
     "text": {
      "type": "string"
     },
     "width": {
      "type": "number"
     },
     "height": {
      "type": "number"
     },
     "blocks": {
      "type": "array",
      "items": {
       "$ref": "#/components/schemas/Block"
      }
     }
    }
   },
   "Block": {
    "type": "object",
    "required": [
     "type",
     "content",
     "page_number"
    ],
    "properties": {
     "type": {
      "type": "string",
      "description": "`text`, `title`, `section_header`, `table`, `list`, ..."
     },
     "content": {
      "type": "string"
     },
     "text": {
      "type": "string"
     },
     "page_number": {
      "type": "integer"
     },
     "confidence": {
      "type": "number"
     },
     "bbox": {
      "type": "object",
      "required": [
       "x0",
       "y0",
       "x1",
       "y1"
      ],
      "properties": {
       "x0": {
        "type": "number"
       },
       "y0": {
        "type": "number"
       },
       "x1": {
        "type": "number"
       },
       "y1": {
        "type": "number"
       }
      }
     }
    }
   }
  }
 }
}