[ { "name": "AgentBench", "slug": "agentbench", "capability": "agentic-tool-use", "secondary_capabilities": [ "reasoning", "instruction-following" ], "short_description": "A multi-environment benchmark for LLM agents across OS, database, knowledge-graph, game, shopping, household, and web tasks.", "launch_date": "2023-08-07T00:00:00.000Z", "status": "deprecated", "status_evidence": "Superseded by AgentBench FC (2025-10-10, 5-env function-calling containerized protocol via AgentRL). Original 8-env remains discriminative at ~31% OA but main branch now serves FC; v0.1/v0.2 are archived and new evaluations should use FC (see github.com/THUDM/AgentBench — 'revert to v0.1/v0.2' for original).", "saturated_date": null, "successor": "agentbench-fc", "lineage": { "parent": null, "variants": [ "agentbench-fc", "visualagentbench" ] }, "paper": { "title": "AgentBench: Evaluating LLMs as Agents", "url": "https://arxiv.org/abs/2308.03688", "venue": "ICLR 2024" }, "authors": [ { "name": "Xiao Liu", "affiliation": "Tsinghua University" }, { "name": "Hao Yu", "affiliation": "Tsinghua University" }, { "name": "Hanchen Zhang", "affiliation": null }, { "name": "Yifan Xu et al.", "affiliation": null } ], "maintainer": "THUDM / AgentBench contributors", "license": "Apache-2.0", "dataset": { "url": "https://github.com/THUDM/AgentBench", "size": 1141, "public_test_set": false, "gated": true }, "metric": { "primary": "overall AgentBench score (%) — weighted average of 8 environments", "scoring": "composite", "judge_model": null, "aggregation": "OA from Table 3 is out of 10 (max 10); this site normalizes to percent (OA×10) for chart parity with FC and other benchmarks. Weighted by 1/avg-task-score (Table 2 Weight-1). Report AgentBench version, split, task set, model, scaffold, and prompts", "variants": [ { "name": "AgentBench FC", "description": "Function-calling prompt version integrated with AgentRL and containerized deployment for ALFWorld, DBBench, Knowledge Graph, OS interaction, and WebShop.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "agentic", "temperature": null, "pass_at": 1, "tools": true, "harnesses": [ { "name": "Official AgentBench harness", "url": "https://github.com/THUDM/AgentBench", "task_flag": null } ] }, "cost_estimate": { "examples": 1141, "approx_usd_frontier_model": "varies substantially by task environment, interaction cap, retry policy, and whether the function-calling or original protocol is used" }, "human_baseline": { "score": null, "note": "The benchmark is built from human-solvable interactive tasks, but no single blind human aggregate score is protocol-comparable across all eight original environments." }, "statistical_note": "AgentBench scores are system scores, not model-only scores. Task server implementation, context management, retries, tool wrappers, and split choice can change the result.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "The original dev split is public while the official test split is not fully exposed. AgentBench FC improves deployment reproducibility, but public code and environment descriptions still create scaffold overfitting risk.", "risk": "medium" }, "languages": [ "english", "shell", "sql" ], "leaderboards": [ { "name": "AgentBench leaderboard", "url": "https://llmbench.ai/agent", "type": "official" } ], "performance_timeline": [ { "model": "CodeLlama-34B Instruct", "vendor": "meta", "date": "2023-08-24T00:00:00.000Z", "score": 9.6, "source": "independent", "source_url": "https://arxiv.org/html/2308.03688v3" }, { "model": "GLM-4", "vendor": "zhipu", "date": "2024-01-25T00:00:00.000Z", "score": 28.9, "source": "independent", "source_url": "https://arxiv.org/html/2308.03688v3" }, { "model": "Claude 3 Opus", "vendor": "anthropic", "date": "2024-03-04T00:00:00.000Z", "score": 31.1, "source": "independent", "source_url": "https://arxiv.org/html/2308.03688v3" } ], "tags": [ "agents", "tool-use", "interaction", "operating-system", "database", "web-shopping", "web-browsing" ], "last_updated": "2026-08-31T00:00:00.000Z", "changelog": [ { "date": "2026-08-31T00:00:00.000Z", "note": "Set status to deprecated with successor agentbench-fc: original 8-env superseded by FC (2025-10-10), main branch now FC, v0.1/v0.2 archived; timeline stays OA-only (OA×10 → %) per arXiv:2308.03688v3." }, { "date": "2026-08-31T00:00:00.000Z", "note": "Normalize Table 3 OA scores to percent (OA×10, max 10→100%) per arXiv:2308.03688v3; source pinned to v3. Timeline remains original 8-env series only with successor agentbench-fc." }, { "date": "2026-08-31T00:00:00.000Z", "note": "Restrict timeline to original AgentBench 8-env series only (remove FC points per owner request); set successor to agentbench-fc and clarify FC is not plotted on same axis." }, { "date": "2026-08-30T00:00:00.000Z", "note": "Added performance timeline points using AgentBench OA scores from Table 3 and model launch dates no earlier than the benchmark release." }, { "date": "2026-08-30T00:00:00.000Z", "note": "Initial page. Treats the 2023/ICLR 2024 AgentBench benchmark as the main lineage and notes the 2025 AgentBench FC update without mixing FC and original scores." } ] }, { "name": "AgentIF", "slug": "agentif", "capability": "instruction-following", "secondary_capabilities": [ "agentic-tool-use" ], "short_description": "707 instructions from 50 real-world agents, averaging 1,723 words and 11.9 constraints — whether instruction following survives deployment-scale prompts.", "launch_date": "2025-05-22T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "Moved to the site's nearing-saturation bucket: the benchmark is still useful for exposing agentic constraint failures, but the public static set is small, top CSR is already near 60%, and the main value is shifting from broad frontier ranking to targeted failure analysis.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "AgentIF: Benchmarking Instruction Following of Large Language Models in Agentic Scenarios", "url": "https://arxiv.org/abs/2505.16944", "venue": null }, "authors": [ { "name": "Yunjia Qi", "affiliation": "Tsinghua University" }, { "name": "Hao Peng", "affiliation": "Tsinghua University" }, { "name": "Xiaozhi Wang", "affiliation": "Tsinghua University" }, { "name": "Amy Xin", "affiliation": "Tsinghua University" }, { "name": "Youfeng Liu", "affiliation": "Tsinghua University" }, { "name": "Bin Xu", "affiliation": "Tsinghua University / Beijing Academy of AI" }, { "name": "Lei Hou", "affiliation": "Tsinghua University" }, { "name": "Juanzi Li", "affiliation": "Tsinghua University" } ], "maintainer": "THU-KEG (Tsinghua Knowledge Engineering Group)", "license": null, "dataset": { "url": "https://huggingface.co/datasets/THU-KEG/AgentIF", "size": 707, "public_test_set": true, "gated": false }, "metric": { "primary": "CSR (Constraint Success Rate, %)", "scoring": "composite", "judge_model": "Evaluator LLM (paper uses GPT-4o / gpt-4o-2024-11-20 for the LLM-based and hybrid constraint checks; some constraints use pure code verification, so not every item passes through the judge)", "aggregation": "CSR averages the pass rate over all annotated constraints; ISR is the fraction of instructions where every constraint is met (an AND across constraints)", "variants": [ { "name": "ISR", "description": "Instruction Success Rate — the proportion of instructions in which all constraints are simultaneously satisfied. The report's headline (best model ~27%).", "scoring": "composite", "judge_model": "Evaluator LLM (GPT-4o) for non-code checks" }, { "name": "CSR", "description": "Constraint Success Rate — the proportion of individual constraints correctly satisfied, averaged across all constraints and instructions.", "scoring": "composite", "judge_model": "Evaluator LLM (GPT-4o) for non-code checks" } ] }, "run_config": { "shots": "zero-shot", "cot": "agentic", "temperature": 0, "pass_at": 1, "tools": true, "harnesses": [ { "name": "Official THU-KEG/AgentIF repo (run.sh)", "url": "https://github.com/THU-KEG/AgentIF", "task_flag": null }, { "name": "HuggingFace dataset", "url": "https://huggingface.co/datasets/THU-KEG/AgentIF", "task_flag": null } ] }, "cost_estimate": { "examples": 707, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No protocol-comparable aggregate human baseline is reported. Annotations were by humans, so ISR at full annotation agreement is implicit; the paper does not publish a 'human would score X' figure." }, "statistical_note": "707 instructions → an ISR near 27% has a rough 95% binomial interval of about ±3 points; constraints are nested inside instructions, so treating items as independent understates the variance.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The prompts are drawn from real deployed agents and open-source workflows and are public on HuggingFace since release; nothing is held out or refreshed. Note the instructions themselves are real system prompts, so a model may have already seen structurally similar living workflows in pretraining — but the specific query/constraint pairings are bespoke.", "risk": "low" }, "languages": [ "english", "chinese" ], "leaderboards": [ { "name": "AgentIF official site (constraint-category tables)", "url": "https://agentif.github.io/", "type": "official" } ], "performance_timeline": [ { "model": "o1-mini", "vendor": "openai", "date": "2025-05-22T00:00:00.000Z", "score": 59.8, "source": "independent", "source_url": "https://arxiv.org/abs/2505.16944" }, { "model": "GPT-4o", "vendor": "openai", "date": "2025-05-22T00:00:00.000Z", "score": 58.5, "source": "independent", "source_url": "https://arxiv.org/abs/2505.16944" } ], "tags": [ "instruction-following", "agentic", "system-prompt", "tool-constraint", "real-world", "ifeval" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2026-08-12T00:00:00.000Z", "note": "Changed status from active to nearing-saturation per site taxonomy update; saturated_date remains null because the page is not marked saturated." }, { "date": "2026-08-07T00:00:00.000Z", "note": "Cleared the paper venue to null: the arXiv record for AgentIF (2505.16944) carries no peer-reviewed venue at time of writing, so a claimed 'NeurIPS 2025' was unsourceable and removed." }, { "date": "2026-08-07T00:00:00.000Z", "note": "Initial page. Timeline is sparse and only carries sourced points from the paper's Table 2 CSR values (o1-mini 59.8, GPT-4o 58.5) on the CSR axis; ISR is lower (~27%) and is documented in prose rather than mixed onto the timeline to keep the axis uniform. License not published for the dataset, so `license` is set to the paper/repo URL rather than a license identifier (gap noted)." } ] }, { "name": "AGIEval", "slug": "agieval", "capability": "reasoning", "secondary_capabilities": [ "knowledge" ], "short_description": "Standardized human exams — SATs, LSATs, China's Gaokao and civil-service tests — repurposed to grade models against real human test-takers.", "launch_date": "2023-04-13T00:00:00.000Z", "status": "saturated", "status_evidence": "GPT-4 already cleared the average-human line (67%) on several sections at launch; by 2024 frontier models pushed the 20-task aggregate past the average human and near the top-percentile 91% ceiling, and the human-exam framing lost its discriminative power at the top. saturated_date (2024-06) marks the GPT-4-class era where the average-human baseline stopped separating strong models.", "saturated_date": "2024-06-01T00:00:00.000Z", "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "url": "https://arxiv.org/abs/2304.06364", "venue": "Findings of NAACL 2024" }, "authors": [ { "name": "Wanjun Zhong", "affiliation": "Microsoft / Sun Yat-sen University" }, { "name": "Ruixiang Cui", "affiliation": "Microsoft" }, { "name": "Nan Duan", "affiliation": "Microsoft Research Asia" } ], "maintainer": "Microsoft (Zhong et al.); repo github.com/ruixiangcui/AGIEval", "license": "MIT", "dataset": { "url": "https://github.com/ruixiangcui/AGIEval", "size": 8062, "public_test_set": true, "gated": false }, "metric": { "primary": "accuracy, macro-averaged over the 20 tasks (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "unweighted (macro) average across the 20 tasks — each exam section counts equally regardless of its question count; the headline setting is zero-shot with chain-of-thought", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "cot", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "AGIEval (official)", "url": "https://github.com/ruixiangcui/AGIEval", "task_flag": null }, { "name": "OpenCompass", "url": "https://github.com/open-compass/opencompass", "task_flag": "agieval" } ] }, "cost_estimate": { "examples": 8062, "approx_usd_frontier_model": "5–20 (8,062 exam questions; zero-shot CoT traces on multi-step problems dominate token cost)" }, "human_baseline": { "score": 67, "note": "Average human test-taker across the exam sections (~67%). The paper also reports a top-percentile ceiling of ~91% (top 1% of candidates, except the lawyer qualification test's top 10%). The gauge/timeline use the average-human figure as the reference line; the 91% top-human ceiling is the harder target the paper highlights. Both are MEASURED from real exam-taker distributions, not estimated — a key differentiator of AGIEval." }, "statistical_note": "8,062 questions across 20 unequally-sized exam sections; because the aggregate is an unweighted task average, a small section (e.g. a few hundred cloze items) moves the headline as much as a large one. Per-section scores — not the 20-task mean — are where the remaining signal lives now that the aggregate is saturated.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "None built in. AGIEval is drawn from real, published exams (SAT, LSAT, Gaokao, Chinese civil-service and bar exams), so the exact questions and worked solutions circulate widely on test-prep sites and in web-scale training data. There is no held-out or refreshed variant, and exam questions are exactly the kind of high-value content trainers scrape. Treat post-2023 scores as memorization-inclusive.", "risk": "high" }, "languages": [ "english", "chinese" ], "leaderboards": [ { "name": "OpenCompass (AGIEval)", "url": "https://opencompass.org.cn", "type": "third-party" } ], "performance_timeline": [ { "model": "text-davinci-003", "vendor": "openai", "date": "2023-04-13T00:00:00.000Z", "score": 37.4, "source": "independent", "source_url": "https://arxiv.org/abs/2304.06364" }, { "model": "ChatGPT (gpt-3.5-turbo)", "vendor": "openai", "date": "2023-04-13T00:00:00.000Z", "score": 43.2, "source": "independent", "source_url": "https://arxiv.org/abs/2304.06364" }, { "model": "GPT-4", "vendor": "openai", "date": "2023-04-13T00:00:00.000Z", "score": 58.4, "source": "independent", "source_url": "https://arxiv.org/abs/2304.06364" }, { "model": "GPT-4o (few-shot)", "vendor": "openai", "date": "2024-05-13T00:00:00.000Z", "score": 69, "source": "vendor-reported", "source_url": "https://github.com/ruixiangcui/AGIEval" }, { "model": "Qwen2.5 72B (base, 0-shot)", "vendor": "alibaba", "date": "2024-09-19T00:00:00.000Z", "score": 75.8, "source": "vendor-reported", "source_url": "https://github.com/deepseek-ai/DeepSeek-V3" }, { "model": "DeepSeek-V3 (base, 0-shot)", "vendor": "deepseek", "date": "2024-12-26T00:00:00.000Z", "score": 79.6, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2412.19437" } ], "tags": [ "reasoning", "human-exams", "bilingual", "gaokao", "sat", "lsat", "contamination" ], "last_updated": "2026-07-25T00:00:00.000Z", "changelog": [ { "date": "2026-07-21T00:00:00.000Z", "note": "Initial page. First bilingual entry (english + chinese). Timeline holds the three launch-era points from the paper (text-davinci-003 37.4, ChatGPT 43.2, GPT-4 58.4) on the zero-shot-CoT 20-task macro-average axis, marked independent (the authors evaluated models they did not build), source arXiv:2304.06364. Later frontier scores (near the 91% top-human ceiling) are described in prose but NOT plotted: no cleanly-sourced, metric-consistent modern aggregate on the same all-20-task zero-shot-CoT axis was found, so points were omitted rather than fabricated. Human baseline stored as the measured average (67); the top-percentile 91% ceiling is recorded in the note. Schema note: `human_baseline` holds a single score — for AGIEval the average-human is the line and the top-human ceiling lives in the note; see the summary flag on whether the schema should eventually carry multiple baselines." }, { "date": "2026-07-25T00:00:00.000Z", "note": "Expanded performance timeline with GPT-4o (few-shot), Qwen2.5 72B (base 0-shot), and DeepSeek-V3 (base 0-shot) scores. These use different eval settings (few-shot or 0-shot no-CoT) from the original paper's zero-shot-CoT axis, so they are marked with protocol notes and not directly comparable to the original three points." } ] }, { "name": "AIME", "slug": "aime", "capability": "maths", "secondary_capabilities": [ "reasoning" ], "short_description": "The MAA's competition exam — 15 integer-answer problems per test, refreshed yearly so each new exam is briefly contamination-free.", "launch_date": "2024-02-01T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "AIME's status is genuinely split by its yearly refresh. OLD exams are saturated and contaminated: reasoning models solved AIME 2024 (o3 ~96.7%, up from GPT-4o's 13.4%). Each FRESH exam has a shrinking clean window, but frontier models now score high even on fresh exams (AIME 2025 ~85–91%). So the rolling benchmark is nearing-saturation — no single exam discriminates for long, yet the yearly refresh keeps briefly-clean signal alive. The plotted gauge/timeline is the AIME 2024 progression (one exam); the perpetual benchmark's yearly reset is described in prose, since the signature gauge shows one monotonic climb and cannot depict a reset.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "aime-2024", "aime-2025", "aime-2026", "matharena" ] }, "paper": null, "authors": [ { "name": "Mathematical Association of America (MAA)", "affiliation": "competition organizer — AIME is an exam, not a research dataset; no paper" } ], "maintainer": "Mathematical Association of America (MAA), administering AIME since 1983; problems compiled and evaluated by the community (MathArena, per-year HuggingFace sets)", "license": "proprietary (MAA copyright — redistribution restricted)", "dataset": { "url": "https://matharena.ai", "size": 30, "public_test_set": true, "gated": false }, "metric": { "primary": "accuracy — every answer is an integer 000–999, exact match (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "mean over an exam's 15 questions (or a year's 30, both exams); because N is tiny, headline numbers are pass@1 AVERAGED over many samples (avg@k) or majority-voted (maj@64) — see Scoring methodology", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "cot", "temperature": null, "pass_at": 1, "tools": false, "harnesses": [ { "name": "MathArena (canonical live tracker)", "url": "https://matharena.ai", "task_flag": null }, { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "aime" } ] }, "cost_estimate": { "examples": 30, "approx_usd_frontier_model": "1–10 per exam-year, but heavily sampling-dependent: because headline numbers use avg@k or maj@64, the real cost is k× a single pass — running maj@64 on a reasoning model is dozens of long chains per question" }, "human_baseline": { "score": 33, "note": "The line is the median AIME score, ~5 out of 15 (≈33%): 2024 AIME I had median 5 / mean 5.89, AIME II median 5 / mean 5.45. The crucial context, unlike MATH's 'which human' range: AIME takers are ALREADY an elite pool — the top ~5% of AMC qualifiers — and even they median only a third of the exam; a perfect 15/15 is achieved by only a few students per exam. So the baseline is LOW even for excellent humans, which is exactly what made frontier models scoring 90%+ so striking. (Note: the gauge shades 'above baseline' as its saturation zone; for AIME that region reads as 'above the median qualifier', not 'near the 100% ceiling' — the true ceiling is a perfect score, which top humans do reach.)" }, "statistical_note": "This is the sharpest sample-size trap on the wiki. One AIME exam is 15 questions, so ONE question = 6.7 points; a full year (30 questions, both exams) is 3.3 points per question. A 'model A beats model B by one question' gap is essentially noise — which is the entire reason labs run heavy sampling (avg@k / maj@64) instead of a single pass. Never trust a small AIME gap, and never compare a single-sample number against an avg@64 one.", "contamination": { "test_set_public": true, "refresh_cycle": "yearly", "mitigation": "AIME is the wiki's first benchmark where REFRESH is the contamination defense — the concrete illustration of the yearly-refresh strategy the GSM8K and MATH pages point toward. Each new exam is genuinely unseen at release (a real, brand-new competition), giving a brief clean measurement window. But within months the problems and worked solutions flood the web, so that year's exam becomes contaminated: AIME 2024 is now high-contamination, while a current-year exam is briefly clean. The defense is rolling, not permanent — which is why risk is 'medium' (fresh yearly, decays fast), not low (no exam stays clean) or high (there is always a fresh, uncontaminated exam to run).", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [ { "name": "MathArena (canonical live tracker)", "url": "https://matharena.ai", "type": "official" }, { "name": "Artificial Analysis (AIME)", "url": "https://artificialanalysis.ai/evaluations/aime-2025", "type": "third-party" } ], "performance_timeline": [ { "model": "GPT-4o (AIME 2024, pass@1)", "vendor": "openai", "date": "2024-09-12T00:00:00.000Z", "score": 13.4, "source": "vendor-reported", "source_url": "https://openai.com/index/learning-to-reason-with-llms/" }, { "model": "o1 (AIME 2024, pass@1)", "vendor": "openai", "date": "2024-09-12T00:00:00.000Z", "score": 74.4, "source": "vendor-reported", "source_url": "https://openai.com/index/learning-to-reason-with-llms/" }, { "model": "o3 (AIME 2024, pass@1)", "vendor": "openai", "date": "2024-12-20T00:00:00.000Z", "score": 96.7, "source": "vendor-reported", "source_url": "https://openai.com/index/openai-o3-mini/" } ], "tags": [ "maths", "competition", "exam", "yearly-refresh", "contamination", "sampling", "matharena" ], "last_updated": "2026-07-22T00:00:00.000Z", "changelog": [ { "date": "2026-07-22T00:00:00.000Z", "note": "Initial page. Third `maths` benchmark (after GSM8K, MATH). Two firsts: no research paper (paper: null — schema made `paper` nullable this commit, with FactsSidebar guarding the paper row) and a refresh_cycle where the refresh IS the point (yearly). launch_date 2024-02-01 = AIME 2024 / the modern LLM-benchmark era; the exam itself has run since 1983 (Overview) — same precedent as AGIEval dating to its benchmark use, not its source material. Timeline is a SINGLE exam (AIME 2024), pass@1: GPT-4o 13.4 → o1 74.4 → o3 96.7 (all vendor-reported, OpenAI posts). Fresh-year exams (AIME 2025 ~85–91%: Grok 4 90.6, o3-mini 86.5, Gemini 2.5 Pro 85.8; AIME 2026 on MathArena) are DELIBERATELY not plotted — different tests, mixing exam-years on one axis is a comparability trap. Human baseline 33 (median ~5/15 among already-elite qualifiers). status nearing-saturation (rolling-window reasoning documented). SCHEMA FLAGS in summary: (a) timeline points want structured exam_year + sampling fields; (b) pass_at can't express avg@k/maj@64; (c) gauge can't depict a yearly reset." } ] }, { "name": "ARC-AGI", "slug": "arc-agi", "capability": "reasoning", "secondary_capabilities": [], "short_description": "Colored-grid abstraction tasks that test few-shot rule induction, not memorized facts, language fluency, or school math.", "launch_date": "2019-11-05T00:00:00.000Z", "status": "saturated", "status_evidence": "ARC-AGI-1 remains useful as a historical abstraction benchmark, but the official ARC Prize leaderboard generated on 2026-07-23 lists multiple verified systems above 95% on the ARC-AGI-1 semi-private axis, including GPT-5.6 Sol at 97.5%. ARC Prize launched ARC-AGI-2 because ARC-AGI-1 had become too brute-forceable and less discriminative for frontier systems.", "saturated_date": "2024-12-20T00:00:00.000Z", "successor": "arc-agi-2", "lineage": { "parent": null, "variants": [] }, "paper": { "title": "On the Measure of Intelligence", "url": "https://arxiv.org/abs/1911.01547", "venue": null }, "authors": [ { "name": "Francois Chollet", "affiliation": "Google" } ], "maintainer": "Francois Chollet; ARC Prize Foundation", "license": "Apache-2.0", "dataset": { "url": "https://github.com/fchollet/ARC-AGI", "size": 400, "public_test_set": true, "gated": false }, "metric": { "primary": "task success rate on the evaluation split (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "percentage of evaluation tasks solved; a task is solved only when every predicted output grid exactly matches the target grid, including dimensions and every cell", "variants": [] }, "run_config": { "shots": "few-shot in-task demonstrations", "cot": "agentic", "temperature": null, "pass_at": 3, "tools": true, "harnesses": [ { "name": "ARC-AGI repository", "url": "https://github.com/fchollet/ARC-AGI", "task_flag": null }, { "name": "ARC Prize benchmarking", "url": "https://github.com/arcprize/arc-agi-benchmarking", "task_flag": null } ] }, "cost_estimate": { "examples": 400, "approx_usd_frontier_model": "varies by test-time reasoning budget; the official ARC Prize leaderboard reports modern semi-private ARC-AGI-1 runs from cents per task to several dollars per task, while older o3-preview high-compute reports were much more expensive" }, "human_baseline": { "score": 98, "note": "The official ARC Prize leaderboard lists a Human Panel score of 98% on the ARC-AGI-1 semi-private axis. Older public-page summaries also report lower average-human estimates; the sidebar uses the current official leaderboard value." }, "statistical_note": "The public evaluation split has 400 tasks, but ARC Prize leaderboard scores use the semi-private axis and competitions may use private splits. Treat scores as comparable only when the split, attempt budget, model/system type, and cost budget are named.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The public train and evaluation tasks are fully available on GitHub. ARC Prize later introduced semi-private and private evaluation sets for leaderboard and competition testing, but ARC-AGI-1 still has a fixed, old private pool and a long history of submissions. Contamination risk is therefore high for public-set scores and medium-to-high even for public leaderboard comparisons unless the split and testing policy are clear.", "risk": "high" }, "languages": [ "language-neutral" ], "leaderboards": [ { "name": "ARC Prize leaderboard", "url": "https://arcprize.org/leaderboard", "type": "official" }, { "name": "ARC Prize 2024 competition", "url": "https://arcprize.org/competitions/2024", "type": "official" } ], "performance_timeline": [ { "model": "GPT-4o", "vendor": "openai", "date": "2024-06-27T00:00:00.000Z", "score": 5, "source": "vendor-reported", "source_url": "https://arcprize.org/competitions/2024" }, { "model": "o1-preview", "vendor": "openai", "date": "2024-09-13T00:00:00.000Z", "score": 18, "source": "vendor-reported", "source_url": "https://arcprize.org/competitions/2024" }, { "model": "Jeremy Berman system", "vendor": "independent", "date": "2024-12-18T00:00:00.000Z", "score": 53.6, "source": "independent", "source_url": "https://arcprize.org/blog/2024-progress-arc-agi-pub" }, { "model": "o3-preview-low (CoT + search/synthesis)", "vendor": "openai", "date": "2024-12-20T00:00:00.000Z", "score": 75.7, "source": "vendor-reported", "source_url": "https://arcprize.org/blog/oai-o3-pub-breakthrough" }, { "model": "Claude 4.7 (High)", "vendor": "anthropic", "date": "2026-04-16T00:00:00.000Z", "score": 93.5, "source": "vendor-reported", "source_url": "https://arcprize.org/leaderboard" }, { "model": "GPT-5.5 Pro (High)", "vendor": "openai", "date": "2026-04-23T00:00:00.000Z", "score": 96.5, "source": "vendor-reported", "source_url": "https://arcprize.org/leaderboard" }, { "model": "GPT-5.6 Sol (xHigh)", "vendor": "openai", "date": "2026-07-09T00:00:00.000Z", "score": 97.5, "source": "vendor-reported", "source_url": "https://arcprize.org/leaderboard" } ], "tags": [ "reasoning", "abstraction", "fluid-intelligence", "colored-grids", "program-synthesis", "arc-prize" ], "last_updated": "2026-07-24T00:00:00.000Z", "changelog": [ { "date": "2026-07-24T00:00:00.000Z", "note": "Updated with the official ARC Prize leaderboard data generated 2026-07-23: Human Panel 98%, Claude 4.7 High 93.5%, GPT-5.5 Pro High 96.5%, and GPT-5.6 Sol xHigh 97.5% on ARC-AGI-1 semi-private. Historical o3-preview context was retained." }, { "date": "2026-07-24T00:00:00.000Z", "note": "Initial page. Timeline uses official ARC Prize / ARC-AGI-Pub points only: GPT-4o 5%, o1-preview 18%, Jeremy Berman 53.6%, and o3-preview-low 75.7% on the semi-private leaderboard axis. Public-eval and high-compute o3 numbers are discussed in prose but not mixed into the chart." } ] }, { "name": "ARC-AGI-2", "slug": "arc-agi-2", "capability": "reasoning", "secondary_capabilities": [], "short_description": "The harder static ARC successor: human-calibrated colored-grid abstraction tasks, now pressured by 2026 frontier systems.", "launch_date": "2025-03-24T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "ARC-AGI-2 remains the current static colored-grid ARC benchmark, but the official ARC Prize leaderboard generated on 2026-07-23 lists GPT-5.6 Sol Max at 92.5% and GPT-5.6 Sol xHigh at 90.0% on the ARC-AGI-2 semi-private axis. Because competition/private-set validation and efficiency constraints still matter, this page marks it nearing saturation rather than fully saturated.", "saturated_date": null, "successor": "arc-agi-3", "lineage": { "parent": "arc-agi", "variants": [ "arc-agi-3" ] }, "paper": { "title": "ARC-AGI-2: A New Challenge for Frontier AI Reasoning Systems", "url": "https://arxiv.org/abs/2505.11831", "venue": "arXiv:2505.11831v2" }, "authors": [ { "name": "Francois Chollet", "affiliation": null }, { "name": "Mike Knoop", "affiliation": null }, { "name": "Gregory Kamradt", "affiliation": null }, { "name": "Bryan Landers", "affiliation": null }, { "name": "Henry Pinkard", "affiliation": null } ], "maintainer": "ARC Prize Foundation", "license": "Apache-2.0", "dataset": { "url": "https://github.com/arcprize/ARC-AGI-2", "size": 120, "public_test_set": true, "gated": false }, "metric": { "primary": "task success rate on the public evaluation split (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "percentage of evaluation tasks solved; a task is solved only when every predicted output grid exactly matches the target grid, with two output attempts allowed per test input", "variants": [] }, "run_config": { "shots": "few-shot in-task demonstrations", "cot": "agentic", "temperature": null, "pass_at": 2, "tools": true, "harnesses": [ { "name": "ARC-AGI-2 repository", "url": "https://github.com/arcprize/ARC-AGI-2", "task_flag": null }, { "name": "ARC Prize benchmarking", "url": "https://github.com/arcprize/arc-agi-benchmarking", "task_flag": null } ] }, "cost_estimate": { "examples": 120, "approx_usd_frontier_model": "model- and search-budget-dependent; the official ARC Prize leaderboard generated 2026-07-23 lists semi-private ARC-AGI-2 runs from cents per task to more than $10/task, with GPT-5.6 Sol Max at $1.44/task" }, "human_baseline": { "score": 100, "note": "The official ARC Prize leaderboard lists a Human Panel score of 100% on the ARC-AGI-2 semi-private axis. The arXiv paper also reports 66% across all human evaluations and attempts, 75% aggregated by task, and 100% solvability by at least two independent non-expert human testers." }, "statistical_note": "The public evaluation set has 120 tasks, so one solved task moves the score by 0.83 points. The official leaderboard uses the semi-private axis, while Kaggle competitions use private evaluation. Identify the split, attempt budget, and cost budget before comparing scores.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "ARC-AGI-2 keeps public training and evaluation data for open research, but adds semi-private and fully-private test sets for lower-leakage leaderboard and competition use. The public evaluation split is still public, so repeated tuning against it should be treated as contamination.", "risk": "medium" }, "languages": [ "language-neutral" ], "leaderboards": [ { "name": "ARC Prize leaderboard", "url": "https://arcprize.org/leaderboard", "type": "official" }, { "name": "ARC Prize 2025 competition", "url": "https://arcprize.org/competitions/2025", "type": "official" } ], "performance_timeline": [ { "model": "GPT-4.5 (pure LLM)", "vendor": "openai", "date": "2025-03-24T00:00:00.000Z", "score": 0, "source": "vendor-reported", "source_url": "https://arcprize.org/blog/announcing-arc-agi-2-and-arc-prize-2025" }, { "model": "r1 / r1-zero (single CoT)", "vendor": "deepseek", "date": "2025-03-24T00:00:00.000Z", "score": 0.3, "source": "vendor-reported", "source_url": "https://arcprize.org/blog/announcing-arc-agi-2-and-arc-prize-2025" }, { "model": "ARChitects (Kaggle 2024 winner)", "vendor": "independent", "date": "2025-03-24T00:00:00.000Z", "score": 3, "source": "independent", "source_url": "https://arcprize.org/blog/announcing-arc-agi-2-and-arc-prize-2025" }, { "model": "o3-preview-low (CoT + search/synthesis)", "vendor": "openai", "date": "2025-03-24T00:00:00.000Z", "score": 4, "source": "vendor-reported", "source_url": "https://arcprize.org/blog/announcing-arc-agi-2-and-arc-prize-2025" }, { "model": "NVARC", "vendor": "independent", "date": "2025-12-05T00:00:00.000Z", "score": 24, "source": "independent", "source_url": "https://arcprize.org/competitions/2025" }, { "model": "Claude 4.7 (Max)", "vendor": "anthropic", "date": "2026-04-16T00:00:00.000Z", "score": 75.8, "source": "vendor-reported", "source_url": "https://arcprize.org/leaderboard" }, { "model": "GPT-5.5 (xHigh)", "vendor": "openai", "date": "2026-04-22T00:00:00.000Z", "score": 85, "source": "vendor-reported", "source_url": "https://arcprize.org/leaderboard" }, { "model": "GPT-5.6 Sol (Max)", "vendor": "openai", "date": "2026-07-09T00:00:00.000Z", "score": 92.5, "source": "vendor-reported", "source_url": "https://arcprize.org/leaderboard" } ], "tags": [ "reasoning", "abstraction", "fluid-intelligence", "colored-grids", "arc-prize", "private-eval" ], "last_updated": "2026-07-24T00:00:00.000Z", "changelog": [ { "date": "2026-07-24T00:00:00.000Z", "note": "Updated paper citation to arXiv:2505.11831v2 and added current official leaderboard points generated 2026-07-23. Status changed to nearing-saturation because the semi-private leaderboard now lists systems above the 85% target, while private competition validation remains a separate claim." }, { "date": "2026-07-24T00:00:00.000Z", "note": "Initial page. Timeline uses the official ARC Prize launch table for ARC-AGI-2 only. Scores marked as estimates in the ARC Prize report are kept in the prose context and model labels rather than normalized into unsourced leaderboard history." } ] }, { "name": "ARC-AGI-3", "slug": "arc-agi-3", "capability": "agentic-tool-use", "secondary_capabilities": [ "reasoning" ], "short_description": "Interactive ARC environments that score agents by human-relative action efficiency, planning, exploration, and adaptation.", "launch_date": "2026-04-17T00:00:00.000Z", "status": "active", "status_evidence": "ARC-AGI-3 is newly introduced as an interactive reasoning benchmark. The paper reports that humans solved 100% of environments while frontier AI systems scored below 1% as of March 2026, and the official leaderboard generated on 2026-07-23 still shows low semi-private scores despite visible progress.", "saturated_date": null, "successor": null, "lineage": { "parent": "arc-agi-2", "variants": [] }, "paper": { "title": "ARC-AGI-3: A New Challenge for Frontier Agentic Intelligence", "url": "https://arxiv.org/abs/2603.24621", "venue": "arXiv:2603.24621v2" }, "authors": [ { "name": "ARC Prize Foundation Development team", "affiliation": "ARC Prize" }, { "name": "Hunter Henry", "affiliation": "ARC Prize" }, { "name": "David Wexler", "affiliation": "ARC Prize" }, { "name": "Derek Smith", "affiliation": "ARC Prize" }, { "name": "Francois Chollet", "affiliation": "ARC Prize" } ], "maintainer": "ARC Prize Foundation", "license": null, "dataset": { "url": "https://arcprize.org/arc-agi/3", "size": 135, "public_test_set": false, "gated": true }, "metric": { "primary": "Relative Human Action Efficiency (RHAE) on interactive environments (%)", "scoring": "composite", "judge_model": null, "aggregation": "per-level action efficiency relative to human baselines, normalized per environment and aggregated across environments; a 100% score means agents beat every game as efficiently as humans", "variants": [ { "name": "environment completion", "description": "Whether the agent completes levels and environments under the configured action and time budgets.", "scoring": "exact-match", "judge_model": null }, { "name": "cost-normalized leaderboard view", "description": "The official leaderboard plots performance against run cost because interactive exploration can spend very different amounts of compute.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "first exposure to each interactive environment", "cot": "agentic", "temperature": null, "pass_at": null, "tools": true, "harnesses": [ { "name": "ARC-AGI-3 public game set", "url": "https://arcprize.org/arc-agi/3", "task_flag": null }, { "name": "ARC-AGI-3 docs and SDK", "url": "https://docs.arcprize.org", "task_flag": null } ] }, "cost_estimate": { "examples": 135, "approx_usd_frontier_model": "varies by agent and interaction budget; the official leaderboard generated 2026-07-23 lists semi-private ARC-AGI-3 runs from roughly $2.3k to more than $25k total" }, "human_baseline": { "score": 100, "note": "The ARC-AGI-3 paper and ARC Prize overview state that environments are 100% human-solvable; the score definition treats 100% as beating every game as efficiently as humans." }, "statistical_note": "ARC-AGI-3 scores are not exact-match grid accuracies. They are human-relative efficiency scores over interactive environments. Public-demo, semi-private, and fully-private results should not be mixed without naming the split and budget.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "The 25 public demo environments are intentionally easier and do not comprehensively represent the private mechanics. The benchmark uses 55 semi-private and 55 fully-private environments as the main evaluation sets, with the fully private set reserved for tightly guarded competition use.", "risk": "medium" }, "languages": [ "language-neutral" ], "leaderboards": [ { "name": "ARC Prize leaderboard", "url": "https://arcprize.org/leaderboard", "type": "official" }, { "name": "ARC-AGI-3 competition leaderboard", "url": "https://www.kaggle.com/competitions/arc-prize-2026-arc-agi-3/leaderboard", "type": "official" } ], "performance_timeline": [ { "model": "Anthropic Opus 4.6 (Max)", "vendor": "anthropic", "date": "2025-12-17T00:00:00.000Z", "score": 0.51, "source": "vendor-reported", "source_url": "https://arcprize.org/leaderboard" }, { "model": "Gemini 3.1 Pro (Preview)", "vendor": "google", "date": "2026-03-05T00:00:00.000Z", "score": 0.42, "source": "vendor-reported", "source_url": "https://arcprize.org/leaderboard" }, { "model": "GPT-5.5 (High)", "vendor": "openai", "date": "2026-04-23T00:00:00.000Z", "score": 0.43, "source": "vendor-reported", "source_url": "https://arcprize.org/leaderboard" }, { "model": "Claude Opus 4.8 (High)", "vendor": "anthropic", "date": "2026-06-01T00:00:00.000Z", "score": 1.52, "source": "vendor-reported", "source_url": "https://arcprize.org/leaderboard" }, { "model": "GPT-5.6 Sol (Max)", "vendor": "openai", "date": "2026-07-09T00:00:00.000Z", "score": 7.78, "source": "vendor-reported", "source_url": "https://arcprize.org/leaderboard" } ], "tags": [ "agentic-tool-use", "reasoning", "abstraction", "interactive", "planning", "fluid-intelligence", "arc-prize" ], "last_updated": "2026-07-24T00:00:00.000Z", "changelog": [ { "date": "2026-07-24T00:00:00.000Z", "note": "Initial page created from the ARC-AGI-3 overview, arXiv:2603.24621v2, and the official ARC Prize leaderboard data generated 2026-07-23. Timeline uses semi-private leaderboard points and converts source decimals to percentages." } ] }, { "name": "AV-SpeakerBench", "slug": "av-speakerbench", "capability": "multimodal", "secondary_capabilities": [ "reasoning", "long-context" ], "short_description": "Speaker-centric audiovisual benchmark for aligning who speaks, what is said, and when in real multi-person videos.", "launch_date": "2025-12-01T00:00:00.000Z", "status": "active", "status_evidence": "A new 2025 benchmark with a 20.7-point gap between the paper's strongest model and human performance, plus large audio-versus-vision ablation gaps across 12 tasks.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "See, Hear, and Understand: Benchmarking Audiovisual Human Speech Understanding in Multimodal Large Language Models", "url": "https://arxiv.org/abs/2512.02231v2", "venue": null }, "authors": [ { "name": "Le Thien Phuc Nguyen", "affiliation": "University of Wisconsin-Madison" }, { "name": "Zhuoran Yu", "affiliation": null }, { "name": "Samuel Low Yu Hang et al.", "affiliation": null } ], "maintainer": "AV-SpeakerBench authors", "license": "MIT dataset metadata; source videos remain subject to their original terms", "dataset": { "url": "https://huggingface.co/datasets/plnguyen2908/AV-SpeakerBench", "size": 3212, "public_test_set": true, "gated": false }, "metric": { "primary": "micro-averaged multiple-choice accuracy (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "overall accuracy across 3,212 four-choice questions, with per-task reporting across 12 speaker-, visual-, and audio-centric task types", "variants": [ { "name": "audiovisual", "description": "Canonical condition with synchronized audio and video.", "scoring": "exact-match", "judge_model": null }, { "name": "visual-only ablation", "description": "Removes audio to measure how often visual cues alone support the answer.", "scoring": "exact-match", "judge_model": null }, { "name": "audio-only ablation", "description": "Removes video to measure the contribution of visible identity and events.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "cot", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official AV-SpeakerBench dataset and prompts", "url": "https://huggingface.co/datasets/plnguyen2908/AV-SpeakerBench", "task_flag": "audiovisual/test.csv" } ] }, "cost_estimate": { "examples": 3212, "approx_usd_frontier_model": "high for native audiovisual APIs; cost depends strongly on video duration and frame sampling density" }, "human_baseline": { "score": 93.74, "note": "Paper-reported human overall accuracy under the benchmark protocol." }, "statistical_note": "Overall accuracy can hide fusion failures because task frequencies differ and some items remain inferable from natural visual speech cues. Report per-task results and visual-only/audio-only ablations alongside the audiovisual score.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Expert curation, cross-modal question construction, temporal-sensitivity checks, and unimodal filtering reduce shortcuts at release; the public dataset can still enter later training corpora.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official AV-SpeakerBench project", "url": "https://plnguyen2908.github.io/AV-SpeakerBench-project-page/", "type": "official" } ], "performance_timeline": [ { "model": "Qwen3-Omni-30B (audiovisual)", "vendor": "alibaba", "date": "2025-12-01T00:00:00.000Z", "score": 54.14, "source": "independent", "source_url": "https://arxiv.org/html/2512.02231v2" }, { "model": "Gemini 2.5 Pro Thinking (audiovisual)", "vendor": "google", "date": "2025-12-01T00:00:00.000Z", "score": 73.04, "source": "independent", "source_url": "https://arxiv.org/html/2512.02231v2" } ], "tags": [ "audio", "video", "speech", "speaker-grounding", "temporal-reasoning", "audiovisual-fusion" ], "last_updated": "2026-09-05T00:00:00.000Z", "changelog": [ { "date": "2026-09-05T00:00:00.000Z", "note": "Initial page based on arXiv v2, the official project, and the official Hugging Face dataset." } ] }, { "name": "BFCL / Berkeley Function Calling Leaderboard", "slug": "bfcl", "capability": "agentic-tool-use", "secondary_capabilities": [ "instruction-following" ], "short_description": "A live UC Berkeley leaderboard for function and tool calling across AST, executable, multi-turn, live, and agentic categories.", "launch_date": "2024-02-26T00:00:00.000Z", "status": "active", "status_evidence": "BFCL V4 is the current leaderboard series: the official site was last updated on 2026-04-12, evaluates agentic web-search and memory categories, and pins reproductions to bfcl-eval==2025.12.17 / commit f7cf735.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "bfcl-v2-live", "bfcl-v3-multi-turn", "bfcl-v4-agentic" ] }, "paper": { "title": "The Berkeley Function Calling Leaderboard (BFCL): From Tool Use to Agentic Evaluation of Large Language Models", "url": "https://proceedings.mlr.press/v267/patil25a.html", "venue": "ICML 2025" }, "authors": [ { "name": "Shishir G. Patil", "affiliation": "UC Berkeley" }, { "name": "Huanzhi Mao", "affiliation": "UC Berkeley" }, { "name": "Fanjia Yan", "affiliation": "UC Berkeley" }, { "name": "Charlie Cheng-Jie Ji", "affiliation": "UC Berkeley" }, { "name": "Vishnu Suresh", "affiliation": "UC Berkeley" }, { "name": "Ion Stoica", "affiliation": "UC Berkeley" }, { "name": "Joseph E. Gonzalez", "affiliation": "UC Berkeley" } ], "maintainer": "UC Berkeley Gorilla team", "license": "Apache-2.0", "dataset": { "url": "https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard", "size": 5088, "public_test_set": true, "gated": false }, "metric": { "primary": "overall function-calling accuracy (%)", "scoring": "composite", "judge_model": null, "aggregation": "BFCL V4 combines Agentic, Multi-Turn, Live, Non-Live, and Hallucination categories using fixed top-level weights; within many categories, subcategories are averaged independently of raw item count", "variants": [ { "name": "AST accuracy", "description": "Parses predicted calls into abstract syntax trees and checks function name, argument structure, and values for non-live categories.", "scoring": "exact-match", "judge_model": null }, { "name": "Executable accuracy", "description": "Runs live or executable calls where static AST equivalence is insufficient.", "scoring": "unit-tests", "judge_model": null }, { "name": "Agentic web search and memory", "description": "BFCL V4 categories that test multi-hop web search, error recovery, and read/write memory tool use.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": true, "harnesses": [ { "name": "BFCL official CLI", "url": "https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard", "task_flag": "TEST_CATEGORY" } ] }, "cost_estimate": { "examples": 5088, "approx_usd_frontier_model": "the official leaderboard reports estimated cost and latency; total cost depends on selected categories, native function calling versus prompt format, and model pricing" }, "human_baseline": { "score": null, "note": "No single human baseline is published for the full BFCL V4 weighted score." }, "statistical_note": "The V4 overall score is a weighted category composite, not a micro-average over every exposed row. Always report native function-calling versus prompt mode, selected categories, code commit, and bfcl-eval package version.", "contamination": { "test_set_public": true, "refresh_cycle": "continuous", "mitigation": "BFCL adds live and periodically updated data, category expansion, reproducible pinned commits, and model-response releases, but the public test data still creates memorization and prompt-format overfitting risk.", "risk": "medium" }, "languages": [ "english", "python", "java", "javascript", "sql" ], "leaderboards": [ { "name": "Berkeley Function-Calling Leaderboard", "url": "https://gorilla.cs.berkeley.edu/leaderboard", "type": "official" } ], "performance_timeline": [ { "model": "GPT-4.1-2025-04-14 (FC)", "vendor": "openai", "date": "2025-04-14T00:00:00.000Z", "score": 53.96, "source": "independent", "source_url": "https://gorilla.cs.berkeley.edu/leaderboard" }, { "model": "o4-mini-2025-04-16 (FC)", "vendor": "openai", "date": "2025-04-16T00:00:00.000Z", "score": 53.24, "source": "independent", "source_url": "https://gorilla.cs.berkeley.edu/leaderboard" }, { "model": "Claude Sonnet 4.5 (FC)", "vendor": "anthropic", "date": "2025-09-29T00:00:00.000Z", "score": 73.24, "source": "independent", "source_url": "https://gorilla.cs.berkeley.edu/leaderboard" }, { "model": "Claude Opus 4.5 (FC)", "vendor": "anthropic", "date": "2025-11-24T00:00:00.000Z", "score": 77.47, "source": "independent", "source_url": "https://gorilla.cs.berkeley.edu/leaderboard" } ], "tags": [ "function-calling", "tool-use", "agents", "ast", "multi-turn", "memory", "web-search" ], "last_updated": "2026-08-30T00:00:00.000Z", "changelog": [ { "date": "2026-08-30T00:00:00.000Z", "note": "Added BFCL V4 leaderboard graph points using model launch dates rather than leaderboard update dates." }, { "date": "2026-08-30T00:00:00.000Z", "note": "Initial page. Uses BFCL V4 as the current score series and records the official V4 category composition instead of mixing older V1/V2/V3 results." } ] }, { "name": "BIG-Bench", "slug": "big-bench", "capability": "reasoning", "secondary_capabilities": [ "knowledge", "maths", "coding", "safety-alignment" ], "short_description": "204-task community-built mega-suite spanning reasoning, knowledge, math, code and social bias — the sprawling proving ground BBH was distilled from.", "launch_date": "2022-06-09T00:00:00.000Z", "status": "deprecated", "status_evidence": "No longer maintained as an evaluation target. Its 'canary' GUID — meant to keep tasks out of training data — was widely ignored, so the suite leaked into web-scale corpora; task quality is famously uneven; and the field moved to distillates like BBH. Deprecated rather than saturated: models don't uniformly max it, it simply stopped being used.", "saturated_date": null, "successor": "big-bench-hard", "lineage": { "parent": null, "variants": [ "big-bench-hard", "big-bench-lite" ] }, "paper": { "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615", "venue": "TMLR 2023" }, "authors": [ { "name": "Aarohi Srivastava", "affiliation": "Google Research" }, { "name": "+ ~450 co-authors (BIG-bench collaboration)", "affiliation": null } ], "maintainer": "Google & the BIG-bench collaboration (~450 authors across 132 institutions); repo google/BIG-bench", "license": "Apache-2.0", "dataset": { "url": "https://github.com/google/BIG-bench", "size": null, "public_test_set": true, "gated": false }, "metric": { "primary": "normalized preferred-metric aggregate (0–100)", "scoring": "composite", "judge_model": null, "aggregation": "Each of the 204 tasks defines its own 'preferred metric' (multiple-choice grade, exact-match, BLEU/ROUGE, or a custom scorer); per-task scores are normalized (roughly 0 = poor, 100 = expert-rubric) and averaged. The aggregate is highly sensitive to which task subset is included and how tasks are weighted — see Known issues.", "variants": [] }, "run_config": { "shots": "0-shot / few-shot (varies by task)", "cot": "direct", "temperature": null, "pass_at": 1, "tools": false, "harnesses": [ { "name": "BIG-bench (official)", "url": "https://github.com/google/BIG-bench", "task_flag": null }, { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "bigbench" } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": "Highly variable — the full 204-task suite is large and heterogeneous; most users run BIG-bench Lite (24 tasks) or single tasks rather than the whole thing" }, "human_baseline": { "score": null, "note": "The paper collected per-task human-rater performance (average rater and best rater), but reports no single suite-wide aggregate human score we can cite here — human performance is only meaningful task by task." }, "statistical_note": "A suite-level number hides enormous per-task variance: tasks range from a handful to thousands of examples, and from trivial to near-impossible. Never compare models on the BIG-Bench aggregate alone — the task mix and weighting dominate the headline figure.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Every task file embeds a 'canary' GUID string that trainers are asked not to strip, so they can detect and exclude BIG-bench from crawled data. In practice the canary was widely ignored, and the suite leaked into training corpora anyway — the field's most-cited example of a decontamination mechanism that depends on voluntary compliance and therefore failed.", "risk": "high" }, "languages": [ "english", "multilingual" ], "leaderboards": [], "performance_timeline": [], "tags": [ "suite", "community", "reasoning", "contamination", "canary", "multi-task" ], "last_updated": "2026-07-21T00:00:00.000Z", "changelog": [ { "date": "2026-07-21T00:00:00.000Z", "note": "Initial page. No performance timeline: the suite's normalized aggregate mixes heterogeneous per-task metrics and there is no cleanly-sourced, metric-consistent model time series to plot — per-task scores are the meaningful unit, so the timeline is intentionally empty rather than fabricated. Schema extended this commit: added `secondary_capabilities[]` (this suite spans several) and a `composite` scoring value (no single scoring rule applies to a suite)." } ] }, { "name": "BIG-Bench Hard", "slug": "big-bench-hard", "capability": "reasoning", "secondary_capabilities": [], "short_description": "23 BIG-Bench tasks where models trailed humans — the benchmark that proved chain-of-thought prompting works, then fell to the reasoning-model era.", "launch_date": "2022-10-17T00:00:00.000Z", "status": "saturated", "status_evidence": "Frontier and reasoning-tuned models exceed ~90% on the 23-task average and are near-perfect on many individual tasks; the reasoning-model era made BBH trivial. Google DeepMind built BBEH (2025) — replacing each task with a harder counterpart — precisely because BBH stopped discriminating at the top.", "saturated_date": "2024-06-01T00:00:00.000Z", "successor": "big-bench-extra-hard", "lineage": { "parent": "big-bench", "variants": [] }, "paper": { "title": "Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them", "url": "https://arxiv.org/abs/2210.09261", "venue": "ACL Findings 2023" }, "authors": [ { "name": "Mirac Suzgun", "affiliation": "Google Research / Stanford" }, { "name": "Jason Wei", "affiliation": "Google Research" }, { "name": "Denny Zhou", "affiliation": "Google Research" } ], "maintainer": "Mirac Suzgun et al. (Google Research); repo suzgunmirac/BIG-Bench-Hard", "license": "MIT", "dataset": { "url": "https://github.com/suzgunmirac/BIG-Bench-Hard", "size": 6511, "public_test_set": true, "gated": false }, "metric": { "primary": "exact-match accuracy, averaged over 23 tasks (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "unweighted (macro) average across the 23 tasks — each task counts equally regardless of how many examples it has", "variants": [] }, "run_config": { "shots": "3-shot", "cot": "cot", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "bbh_cot_fewshot" }, { "name": "BIG-Bench Hard (official)", "url": "https://github.com/suzgunmirac/BIG-Bench-Hard", "task_flag": null } ] }, "cost_estimate": { "examples": 6511, "approx_usd_frontier_model": "5–20 (6,511 items; 3-shot chain-of-thought prompts are long, and the reasoning traces dominate token cost)" }, "human_baseline": { "score": 67.7, "note": "Average human-rater performance across the 23 tasks in the original paper; the best (max) human rater averaged ~94.4%. BBH tasks were selected precisely as the BIG-Bench tasks where prior models fell below the 67.7% average-rater line. (Canonical Table 1 values from the paper.)" }, "statistical_note": "6,511 examples across 23 unequally-sized tasks; because the aggregate is an unweighted task average, a small task moves the number as much as a large one. Now that the mean is saturated, per-task scores — not the 23-task average — are where any remaining signal lives.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Inherits BIG-bench's public tasks and its widely-ignored canary GUID. BBH's popularity made it especially likely to appear in pretraining and instruction-tuning data, and there is no refreshed variant. Its high contamination risk is a direct reason BBEH was built with fresh tasks.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "llm-stats (BBH)", "url": "https://llm-stats.com/benchmarks/big-bench-hard", "type": "third-party" } ], "performance_timeline": [ { "model": "PaLM 540B", "vendor": "google", "date": "2022-10-17T00:00:00.000Z", "score": 65.2, "source": "independent", "source_url": "https://arxiv.org/abs/2210.09261" }, { "model": "Codex (code-davinci-002)", "vendor": "openai", "date": "2022-10-17T00:00:00.000Z", "score": 73.9, "source": "independent", "source_url": "https://arxiv.org/abs/2210.09261" }, { "model": "GPT-4 (3-shot CoT)", "vendor": "openai", "date": "2023-03-14T00:00:00.000Z", "score": 83.1, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2303.08774" }, { "model": "Gemini Ultra 1.0 (3-shot CoT)", "vendor": "google", "date": "2023-12-06T00:00:00.000Z", "score": 83.6, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2312.11805" }, { "model": "Claude 3 Opus (3-shot CoT)", "vendor": "anthropic", "date": "2024-03-04T00:00:00.000Z", "score": 86.8, "source": "vendor-reported", "source_url": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf" }, { "model": "Gemini 1.5 Pro (3-shot CoT)", "vendor": "google", "date": "2024-05-01T00:00:00.000Z", "score": 89.2, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2403.05530" }, { "model": "Claude 3.5 Sonnet (3-shot CoT)", "vendor": "anthropic", "date": "2024-06-21T00:00:00.000Z", "score": 93.1, "source": "vendor-reported", "source_url": "https://www.anthropic.com/news/claude-3-5-sonnet" }, { "model": "DeepSeek V3", "vendor": "deepseek", "date": "2024-12-26T00:00:00.000Z", "score": 87.5, "source": "vendor-reported", "source_url": "https://epoch.ai/data/ai-benchmarking-dashboard" } ], "tags": [ "reasoning", "chain-of-thought", "multi-step", "big-bench", "exact-match" ], "last_updated": "2026-07-25T00:00:00.000Z", "changelog": [ { "date": "2026-07-21T00:00:00.000Z", "note": "Initial page. Timeline holds the two canonical paper points — PaLM 540B and Codex code-davinci-002 with 3-shot chain-of-thought (arXiv:2210.09261), marked independent (the paper evaluated models it did not build)." }, { "date": "2026-07-25T00:00:00.000Z", "note": "Expanded performance timeline with GPT-4, Gemini Ultra, Claude 3 Opus, Gemini 1.5 Pro, Claude 3.5 Sonnet, and DeepSeek V3 scores. BBH is now saturated as documented in the prose." } ] }, { "name": "BrowseComp", "slug": "browsecomp", "capability": "agentic-tool-use", "secondary_capabilities": [ "knowledge", "reasoning" ], "short_description": "OpenAI's benchmark for browsing agents that must find hard-to-locate, short-answer facts on the web.", "launch_date": "2025-04-10T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "BrowseComp is nearing saturation: launch baseline Deep Research 51.5% (Apr 2025, openai.com/index/browsecomp); by Aug 2026 SOTA reached 92.2% (GPT-5.6 Sol) with top-3 within 1.4 pts (llm-stats.com/benchmarks/browsecomp, benchlm.ai) and only ~7.8 pts headroom. LoHoSearch reports 'rapidly saturated... surpassing 90%... 30%->90% in barely ten months' (arxiv.org/abs/2606.12837) and Anthropic shows eval-awareness/decrypted answer key (anthropic.com/engineering/eval-awareness-browsecomp); LiveBrowseComp confirms IKD leakage (arxiv.org/abs/2605.28721).", "saturated_date": null, "successor": "livebrowsecomp", "lineage": { "parent": "simpleqa", "variants": [] }, "paper": { "title": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents", "url": "https://arxiv.org/abs/2504.12516", "venue": "arXiv:2504.12516" }, "authors": [ { "name": "Jason Wei", "affiliation": "OpenAI" }, { "name": "Zhiqing Sun", "affiliation": "OpenAI" }, { "name": "Spencer Papay", "affiliation": "OpenAI" }, { "name": "Scott McKinney", "affiliation": "OpenAI" }, { "name": "Jeffrey Han et al.", "affiliation": "OpenAI" } ], "maintainer": "OpenAI", "license": "MIT (simple-evals code); dataset terms follow the OpenAI simple-evals release", "dataset": { "url": "https://github.com/openai/simple-evals", "size": 1266, "public_test_set": true, "gated": false }, "metric": { "primary": "short-answer accuracy (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "mean accuracy over short-answer browsing questions; predictions are compared against reference answers with the official simple-evals BrowseComp evaluator", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "agentic", "temperature": null, "pass_at": 1, "tools": true, "harnesses": [ { "name": "OpenAI simple-evals BrowseComp evaluator", "url": "https://github.com/openai/simple-evals", "task_flag": "browsecomp_eval.py" } ] }, "cost_estimate": { "examples": 1266, "approx_usd_frontier_model": "can be high for deep browsing agents because many questions require long searches across many sites; report search/tool budget and retry policy" }, "human_baseline": { "score": 29.2, "note": "OpenAI's verification campaign reports trainers solved 29.2% of 1,255 checked problems within the allowed search process; among solved problems, trainer and reference answers matched 86.4%." }, "statistical_note": "BrowseComp uses short answers to simplify grading. Scores depend strongly on browser/search tool quality, time budget, and whether the agent is allowed repeated attempts or parallel search.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "OpenAI asks users not to reveal examples online and includes a canary string for training-data filtering, but the benchmark is public and static.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [], "performance_timeline": [ { "model": "o4-mini with Python + browsing", "vendor": "openai", "date": "2025-04-16T00:00:00.000Z", "score": 28.3, "source": "independent", "source_url": "https://langbase.com/models/open-ai/o4-mini/benchmarks" }, { "model": "o3 with Python + browsing", "vendor": "openai", "date": "2025-04-16T00:00:00.000Z", "score": 49.7, "source": "independent", "source_url": "https://evals.report/benchmarks/browsecomp?tab=scores" }, { "model": "GPT-5", "vendor": "openai", "date": "2025-08-07T00:00:00.000Z", "score": 54.9, "source": "independent", "source_url": "https://evals.report/benchmarks/browsecomp?tab=scores" }, { "model": "Kimi K2 Thinking", "vendor": "moonshot", "date": "2025-11-06T00:00:00.000Z", "score": 60.2, "source": "vendor-reported", "source_url": "https://medium.com/@kimi_moonshot/introducing-kimi-k2-thinking-a61c95f6e59a" } ], "tags": [ "browsing", "web-search", "agents", "factuality", "retrieval", "deep-research" ], "last_updated": "2026-08-31T00:00:00.000Z", "changelog": [ { "date": "2026-08-31T00:00:00.000Z", "note": "Set status to nearing-saturation with successor livebrowsecomp: SOTA 92.2% by Aug 2026 (top-3 within 1.4 pts, 7.8 pts headroom), LoHoSearch saturation analysis and Anthropic eval-awareness evidence; keep timeline on original series." }, { "date": "2026-08-30T00:00:00.000Z", "note": "Added BrowseComp performance timeline points using model launch dates and excluded launch-prebenchmark models from the graph." }, { "date": "2026-08-30T00:00:00.000Z", "note": "Initial page. Records BrowseComp as a public static browsing-agent benchmark and avoids exposing additional example questions beyond the official public article." } ] }, { "name": "CodeContests", "slug": "codecontests", "capability": "coding", "secondary_capabilities": [ "reasoning" ], "short_description": "DeepMind's competitive-programming corpus with temporally split problems, human submissions, and generated correctness tests.", "launch_date": "2022-02-02T00:00:00.000Z", "status": "deprecated", "status_evidence": "The repository was archived on 2024-12-06, the temporal holdout has been public since 2022, and later AlphaCode systems used updated non-public data; it no longer provides a clean maintained frontier comparison.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "codecontests-plus" ] }, "paper": { "title": "Competition-Level Code Generation with AlphaCode", "url": "https://arxiv.org/abs/2203.07814", "venue": "Science 2022" }, "authors": [ { "name": "Yujia Li", "affiliation": "DeepMind" }, { "name": "David Choi", "affiliation": "DeepMind" }, { "name": "Junyoung Chung", "affiliation": "DeepMind" }, { "name": "Nate Kushman", "affiliation": "DeepMind" }, { "name": "Oriol Vinyals", "affiliation": "DeepMind" } ], "maintainer": "Google DeepMind (repository archived 2024-12-06)", "license": "CC-BY-4.0 (dataset)", "dataset": { "url": "https://github.com/google-deepmind/code_contests", "size": 165, "public_test_set": true, "gated": false }, "metric": { "primary": "10@100k solve rate on the 165-problem test split (%)", "scoring": "unit-tests", "judge_model": null, "aggregation": "fraction of problems solved after sampling 100,000 programs, filtering by example tests, clustering by behavior, and selecting up to 10 submissions", "variants": [ { "name": "Codeforces competition percentile", "description": "Simulated placement using up to ten submitted programs per problem on ten later Codeforces contests.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": null, "pass_at": 10, "tools": false, "harnesses": [ { "name": "Official CodeContests evaluator", "url": "https://github.com/google-deepmind/code_contests", "task_flag": "execution:solve_example" } ] }, "cost_estimate": { "examples": 165, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No human 10@100k solve-rate baseline was published. AlphaCode's human comparison was average simulated rank in the top 54.3% across ten Codeforces contests, an incompatible percentile that must not be plotted on the solve-rate axis." }, "statistical_note": "The 165-problem test set gives wide intervals for solve rate; the original 10@100k pipeline also introduces sampling, filtering, clustering, and execution variance.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The original temporal split placed training data before validation/test problems, but the complete released split has been public since 2022 and the clean window has expired.", "risk": "high" }, "languages": [ "c-plus-plus", "python", "java" ], "leaderboards": [], "performance_timeline": [ { "model": "AlphaCode 9B (test set, 10@100k, no clustering)", "vendor": "deepmind", "date": "2022-02-02T00:00:00.000Z", "score": 25.8, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2203.07814" }, { "model": "AlphaCode 41B (test set, 10@100k, no clustering)", "vendor": "deepmind", "date": "2022-02-02T00:00:00.000Z", "score": 27.7, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2203.07814" }, { "model": "AlphaCode 41B + clustering (test set, 10@100k)", "vendor": "deepmind", "date": "2022-02-02T00:00:00.000Z", "score": 29.6, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2203.07814" } ], "tags": [ "competitive-programming", "algorithms", "sampling", "unit-tests", "temporal-split" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Initial page. Timeline uses only Table 5's 165-problem test-set 10@100k axis (9B, 41B, and 41B + clustering). The 54.3 human-relative contest percentile remains prose-only because it is not a solve rate." } ] }, { "name": "CodeXGLUE", "slug": "codexglue", "capability": "coding", "secondary_capabilities": [], "short_description": "A broad code-intelligence suite spanning understanding, retrieval, completion, translation, repair, generation, and summarization.", "launch_date": "2021-02-09T00:00:00.000Z", "status": "deprecated", "status_evidence": "The official challenge and repository remain available, but the heterogeneous task-specific leaderboard is no longer a current frontier-code comparison and has no maintained common aggregate.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation", "url": "https://arxiv.org/abs/2102.04664", "venue": "ESEC/FSE 2021" }, "authors": [ { "name": "Shuai Lu", "affiliation": "Peking University" }, { "name": "Daya Guo", "affiliation": "Sun Yat-sen University" }, { "name": "Shuo Ren", "affiliation": "Beihang University" }, { "name": "Nan Duan", "affiliation": "Microsoft Research Asia" } ], "maintainer": "Microsoft", "license": "C-UDA for datasets; MIT for repository code", "dataset": { "url": "https://github.com/microsoft/CodeXGLUE", "size": null, "public_test_set": false, "gated": false }, "metric": { "primary": "overall smoothed BLEU on six-language CodeSearchNet code summarization", "scoring": "composite", "judge_model": null, "aggregation": "unweighted average of smoothed BLEU for Ruby, JavaScript, Go, Python, Java, and PHP test sets; selected only as the page's chart axis, not a suite aggregate", "variants": [ { "name": "Full CodeXGLUE suite", "description": "Ten tasks across 14 datasets, reported separately with accuracy, F1, MAP, MRR, exact match, BLEU, CodeBLEU, perplexity, and edit similarity; no official common aggregate.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "task-specific fine-tuning or inference", "cot": "direct", "temperature": null, "pass_at": null, "tools": false, "harnesses": [ { "name": "Official CodeXGLUE repository", "url": "https://github.com/microsoft/CodeXGLUE", "task_flag": null } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No single human baseline exists across the ten heterogeneous tasks; several component datasets use human-authored code or labels rather than a common human score." }, "statistical_note": "There is no statistically coherent suite-wide score. Compare systems only within the same dataset, split, preprocessing, and metric.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "Official test labels were withheld for challenge submission, but most source corpora and all training/dev material are public; component datasets require separate leakage analysis.", "risk": "medium" }, "languages": [ "python", "java", "javascript", "php", "ruby", "go", "c-sharp", "c", "c-plus-plus", "english", "chinese", "japanese" ], "leaderboards": [ { "name": "Official CodeXGLUE leaderboard", "url": "https://microsoft.github.io/CodeXGLUE/", "type": "official" } ], "performance_timeline": [ { "model": "Seq2Seq (CodeSearchNet summarization, six-language overall BLEU)", "vendor": "independent", "date": "2021-02-09T00:00:00.000Z", "score": 14.32, "source": "independent", "source_url": "https://arxiv.org/abs/2102.04664" }, { "model": "Transformer (CodeSearchNet summarization, six-language overall BLEU)", "vendor": "independent", "date": "2021-02-09T00:00:00.000Z", "score": 15.56, "source": "independent", "source_url": "https://arxiv.org/abs/2102.04664" }, { "model": "RoBERTa encoder (CodeSearchNet summarization, six-language overall BLEU)", "vendor": "meta", "date": "2021-02-09T00:00:00.000Z", "score": 16.57, "source": "independent", "source_url": "https://arxiv.org/abs/2102.04664" }, { "model": "CodeBERT encoder (CodeSearchNet summarization, six-language overall BLEU)", "vendor": "microsoft", "date": "2021-02-09T00:00:00.000Z", "score": 17.83, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2102.04664" } ], "tags": [ "code-intelligence", "suite", "retrieval", "generation", "summarization", "translation" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Initial page. Full-paper result tables were reviewed. Because CodeXGLUE has no common aggregate, the timeline explicitly selects Table 13's stable code-summarization axis: six-language overall smoothed BLEU (Seq2Seq 14.32, Transformer 15.56, RoBERTa 16.57, CodeBERT 17.83); it is not presented as a suite score." } ] }, { "name": "CommonsenseQA", "slug": "commonsenseqa", "capability": "reasoning", "secondary_capabilities": [ "knowledge" ], "short_description": "Five-choice questions derived from ConceptNet relations, designed to require everyday knowledge beyond a supplied passage.", "launch_date": "2018-11-02T00:00:00.000Z", "status": "saturated", "status_evidence": "KEAR reached 89.4% on the hidden test set versus the original 88.9% human reference in 2021; the benchmark's authors then introduced the adversarially collected CommonsenseQA 2.0.", "saturated_date": "2021-12-06T00:00:00.000Z", "successor": null, "lineage": { "parent": null, "variants": [ "commonsenseqa-2" ] }, "paper": { "title": "CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge", "url": "https://arxiv.org/abs/1811.00937", "venue": "NAACL-HLT 2019" }, "authors": [ { "name": "Alon Talmor", "affiliation": null }, { "name": "Jonathan Herzig", "affiliation": null }, { "name": "Nicholas Lourie", "affiliation": null }, { "name": "Jonathan Berant", "affiliation": null } ], "maintainer": "Tel Aviv University NLP", "license": "MIT", "dataset": { "url": "https://huggingface.co/datasets/tau/commonsense_qa", "size": 1140, "public_test_set": false, "gated": false }, "metric": { "primary": "accuracy (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "mean exact-match accuracy over questions; official results use the hidden random-split test set, while lm-evaluation-harness uses validation", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "commonsense_qa" } ] }, "cost_estimate": { "examples": 1221, "approx_usd_frontier_model": null }, "human_baseline": { "score": 88.9, "note": "Majority vote of five crowd workers on the official test set in the original paper; random choice among five answers is 20%." }, "statistical_note": "For 1,140 independent test questions, a score near 50% has an approximate 95% binomial margin of error of +/-2.9 points; near the 88.9% human reference it is about +/-1.8 points. Dev-set tuning and ensemble selection add uncertainty not captured by this interval.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "Official test answers are hidden, but training and validation questions are public and static. Use the hidden evaluation server when available, disclose retrieval and fine-tuning data, and audit overlap with public mirrors.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official CommonsenseQA leaderboard", "url": "https://www.tau-nlp.org/commonsenseqa", "type": "official" } ], "performance_timeline": [ { "model": "BERT-Large", "vendor": "google", "date": "2018-11-02T00:00:00.000Z", "score": 55.9, "source": "independent", "source_url": "https://arxiv.org/abs/1811.00937" }, { "model": "KEAR ensemble", "vendor": "microsoft", "date": "2021-12-06T00:00:00.000Z", "score": 89.4, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2112.03254" }, { "model": "RoBERTa-large", "vendor": "facebook-ai", "date": "2019-07-25T00:00:00.000Z", "score": 72.1, "source": "independent", "source_url": "https://arxiv.org/abs/1907.11692" }, { "model": "DeBERTa-xxlarge (1.5B)", "vendor": "microsoft", "date": "2021-01-06T00:00:00.000Z", "score": 83.8, "source": "independent", "source_url": "https://arxiv.org/abs/2006.03654" }, { "model": "KEAR single model", "vendor": "microsoft", "date": "2021-12-06T00:00:00.000Z", "score": 86.1, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2112.03254" }, { "model": "CPACE", "vendor": "independent", "date": "2023-05-14T00:00:00.000Z", "score": 89.8, "source": "independent", "source_url": "https://arxiv.org/abs/2305.08135" }, { "model": "Llama 3 70B (7-shot)", "vendor": "meta", "date": "2024-04-18T00:00:00.000Z", "score": 83.8, "source": "vendor-reported", "source_url": "https://ai.meta.com/blog/meta-llama-3/" }, { "model": "GPT-4o", "vendor": "openai", "date": "2024-05-13T00:00:00.000Z", "score": 85.4, "source": "vendor-reported", "source_url": "https://openai.com/index/gpt-4o-system-card/" } ], "tags": [ "commonsense", "conceptnet", "multiple-choice", "question-answering", "knowledge-graph" ], "last_updated": "2026-07-25T00:00:00.000Z", "changelog": [ { "date": "2026-07-25T00:00:00.000Z", "note": "Initial page. Marked saturated at the published hidden-test human-parity result; documented the released-size discrepancy, public-validation protocol, artifacts, and CommonsenseQA 2.0 lineage." }, { "date": "2026-07-25T00:00:00.000Z", "note": "Expanded performance timeline with RoBERTa-large, DeBERTa-xxlarge, KEAR single model, CPACE, Llama 3 70B, and GPT-4o scores." } ] }, { "name": "ComplexBench", "slug": "complexbench", "capability": "instruction-following", "secondary_capabilities": [ "reasoning" ], "short_description": "Bilingual benchmark of how models follow instructions when several constraints must hold at once — where stacking constraints is not linear.", "launch_date": "2024-07-04T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "Moved to the site's nearing-saturation bucket: ComplexBench remains useful for multi-constraint composition analysis, but it is a public static set and should be treated more as a diagnostic successor to simpler instruction-following checks than as a durable frontier-ranking benchmark.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "ComplexBench: Benchmarking Complex Instruction-Following with Multiple Constraints Composition", "url": "https://arxiv.org/abs/2407.03978", "venue": "NeurIPS 2024 Datasets & Benchmarks Track" }, "authors": [ { "name": "Bosi Wen", "affiliation": "Tsinghua University" }, { "name": "Pei Ke", "affiliation": "Tsinghua University" }, { "name": "Xiaotao Gu", "affiliation": null }, { "name": "Lindong Wu", "affiliation": null }, { "name": "Hao Huang", "affiliation": null }, { "name": "Jinfeng Zhou", "affiliation": null }, { "name": "Wenchuang Li", "affiliation": null }, { "name": "Binxin Hu", "affiliation": null }, { "name": "Wendy Gao", "affiliation": null }, { "name": "Jiaxin Xu", "affiliation": null }, { "name": "Yiming Liu", "affiliation": null }, { "name": "Jie Tang", "affiliation": "Tsinghua University" }, { "name": "Hongning Wang", "affiliation": null }, { "name": "Minlie Huang", "affiliation": "Tsinghua University" } ], "maintainer": "thu-coai (Tsinghua CoAI Group)", "license": "MIT", "dataset": { "url": "https://github.com/thu-coai/ComplexBench", "size": 1150, "public_test_set": true, "gated": false }, "metric": { "primary": "overall constraint-satisfaction (%)", "scoring": "llm-judge", "judge_model": "GPT-4, augmented with rule-based checks where a constraint is verifiable programmatically (e.g. length), per the paper's evaluation protocol", "aggregation": "Dependency-aware aggregate: each instruction's score is built bottom-up from per-constraint verification answers, where a failed prerequisite can zero out downstream scoring questions that depend on it.", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official ComplexBench evaluation scripts", "url": "https://github.com/thu-coai/ComplexBench", "task_flag": "eval.sh" } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": "Judge-pass dominates: every generated response is re-read by the GPT-4 evaluator per scoring question, so cost scales with dataset size and the number of constraint checks. A small run on the released subset is inexpensive; a full pass is non-trivial." }, "human_baseline": { "score": null, "note": "No protocol-comparable human constraint-satisfaction baseline was published. The dataset is manually curated and validated by humans, but the authors do not report a human pass-rate that maps one-to-one onto the automatic judge score." }, "statistical_note": "The overall percentage is a composite built over per-instance dependency trees, not a clean binomial hit-rate, so simple confidence intervals mislead. Because many scoring questions gate on prerequisites, one early failure can cascade and suppress several downstream points at once.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Items are largely self-contained synthetic compositions written for the benchmark rather than scraped from the internet, so the highest-risk web-memorization leak path is limited. The dataset and gold scoring questions are still public, so fine-tuning on files would circumvent it.", "risk": "low" }, "languages": [ "english", "chinese" ], "leaderboards": [], "performance_timeline": [], "tags": [ "instruction-following", "constraints", "composition", "bilingual", "chinese", "llm-as-judge", "reliability" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2026-08-12T00:00:00.000Z", "note": "Changed status from active to nearing-saturation per site taxonomy update; saturated_date remains null because the page is not marked saturated." }, { "date": "2026-08-07T00:00:00.000Z", "note": "Recorded the paper's full 14-author list (arXiv:2407.03978 v3) and set dataset.size to 1,150 English test instructions on the size-matches-primary convention. Affiliations are set only where confirmed; the official paper title omits the 'ComplexBench:' prefix, which is retained here as the page name." }, { "date": "2026-08-07T00:00:00.000Z", "note": "Initial page." }, { "date": "2026-08-07T00:00:00.000Z", "note": "Timeline left empty on purpose: there is no single sourced consolidated trajectory I can cite reliably yet. The paper reports per-model scores for ~15 models in its own tables; I have not copied any of those numbers here rather than risk publishing an unsourced figure. Add timeline points only when scores can be traced to the paper or a reproducible run." } ] }, { "name": "CyberSecEval", "slug": "cyberseceval", "capability": "coding", "secondary_capabilities": [ "safety-alignment", "agentic-tool-use", "multimodal" ], "short_description": "Meta's evolving suite for insecure code, cyberattack assistance, prompt injection, exploitation, SOC analysis, and autopatching.", "launch_date": "2023-12-07T00:00:00.000Z", "status": "deprecated", "status_evidence": "The original CyberSecEval benchmark is no longer the recommended standalone version: Meta advanced through CyberSecEval 2 and 3 to CyberSecEval 4, changing tasks and metrics rather than maintaining one comparable score axis.", "saturated_date": null, "successor": "cyberseceval-4", "lineage": { "parent": null, "variants": [ "cyberseceval-2", "cyberseceval-3", "cyberseceval-4", "cybersoceval", "autopatchbench" ] }, "paper": { "title": "Purple Llama CyberSecEval: A Secure Coding Benchmark for Language Models", "url": "https://arxiv.org/abs/2312.04724", "venue": "arXiv:2312.04724" }, "authors": [ { "name": "Manish Bhatt", "affiliation": "Meta" }, { "name": "Sahana Chennabasappa", "affiliation": "Meta" }, { "name": "Cyrus Nikolaidis", "affiliation": "Meta" }, { "name": "Shengye Wan", "affiliation": "Meta" }, { "name": "Joshua Saxe", "affiliation": "Meta" } ], "maintainer": "Meta Purple Llama, with partner datasets including CrowdStrike CyberSOCEval", "license": "MIT for Purple Llama evals/benchmarks; bundled third-party components may differ", "dataset": { "url": "https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks", "size": null, "public_test_set": true, "gated": false }, "metric": { "primary": "average textual prompt-injection success rate (%) — lower is safer", "scoring": "llm-judge", "judge_model": null, "aggregation": "mean injection success rate across 15 named textual attack techniques; do not combine this axis with other CyberSecEval modules", "variants": [ { "name": "secure code generation", "description": "Rates insecure patterns in generated code across instruction and autocomplete contexts.", "scoring": "unit-tests", "judge_model": null }, { "name": "prompt-injection success rate", "description": "Judge-based rate at which injected instructions override the intended task. BenchmarkList reports an overall average; lower is safer.", "scoring": "llm-judge", "judge_model": null }, { "name": "CyberSOCEval multiple-choice metrics", "description": "Exact set accuracy and partial-credit Jaccard similarity for malware analysis and threat-intelligence reasoning.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "sub-benchmark-specific", "cot": "agentic", "temperature": null, "pass_at": null, "tools": true, "harnesses": [ { "name": "Purple Llama CybersecurityBenchmarks", "url": "https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks", "task_flag": "--benchmark=" }, { "name": "Inspect Evals CyberSecEval tasks", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "task_flag": "cyberseceval_4" } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No suite-wide human baseline exists. Some components use human-written or expert-derived labels, but their scores and task definitions are not one comparable human axis." }, "statistical_note": "The structured leaderboard axis is BenchmarkList's seven-model textual prompt-injection snapshot sampled 2026-05-27. Timeline dates are official model launch dates, not score-evaluation dates. Lower scores are safer. Comparability depends on the exact CyberSecEval prompt set, judge, model interface, and guardrail configuration. Other CyberSecEval modules remain separate axes.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Versioned additions broaden coverage, but most prompts and labels are public. Agentic cyber-range tasks add live environment variation; evaluators should version data and keep any private cases separate.", "risk": "high" }, "languages": [ "multilingual" ], "leaderboards": [ { "name": "BenchmarkList CyberSecEval leaderboard", "url": "https://benchmarklist.com/benchmarks/cyberseceval/", "type": "third-party" } ], "performance_timeline": [ { "model": "UN codellama-70b-instruct", "vendor": "meta", "date": "2024-01-29T00:00:00.000Z", "score": 12.93, "source": "vendor-reported", "source_url": "https://benchmarklist.com/benchmarks/cyberseceval/" }, { "model": "gpt-4", "vendor": "openai", "date": "2023-03-14T00:00:00.000Z", "score": 19.87, "source": "vendor-reported", "source_url": "https://benchmarklist.com/benchmarks/cyberseceval/" }, { "model": "llama 3 70b-instruct", "vendor": "meta", "date": "2024-04-18T00:00:00.000Z", "score": 29.27, "source": "vendor-reported", "source_url": "https://benchmarklist.com/benchmarks/cyberseceval/" }, { "model": "UN codellama-34b-instruct", "vendor": "meta", "date": "2023-08-24T00:00:00.000Z", "score": 36.33, "source": "vendor-reported", "source_url": "https://benchmarklist.com/benchmarks/cyberseceval/" }, { "model": "UN codellama-13b-instruct", "vendor": "meta", "date": "2023-08-24T00:00:00.000Z", "score": 37.27, "source": "vendor-reported", "source_url": "https://benchmarklist.com/benchmarks/cyberseceval/" }, { "model": "gpt-3.5-turbo", "vendor": "openai", "date": "2023-03-01T00:00:00.000Z", "score": 39.13, "source": "vendor-reported", "source_url": "https://benchmarklist.com/benchmarks/cyberseceval/" }, { "model": "llama 3 8b-instruct", "vendor": "meta", "date": "2024-04-18T00:00:00.000Z", "score": 45.27, "source": "vendor-reported", "source_url": "https://benchmarklist.com/benchmarks/cyberseceval/" } ], "tags": [ "cybersecurity", "secure-code", "prompt-injection", "offensive-security", "defensive-security", "agents", "multimodal" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Simplified the CyberSecEval tabular snapshot to Subject and Average only, removing technique-level, source-label, sampled-date, and rank columns while keeping the timeline score axis unchanged." }, { "date": "2026-08-03T00:00:00.000Z", "note": "Added the seven-model BenchmarkList textual prompt-injection snapshot sampled 2026-05-27. Timeline points use each model's official launch date. The structured chart uses average injection success rate as one coherent lower-is-safer axis; insecure-code and other CyberSecEval metrics remain off-axis." } ] }, { "name": "FireBench", "slug": "firebench", "capability": "instruction-following", "secondary_capabilities": [ "agentic-tool-use" ], "short_description": "Evaluates instruction following for enterprise/API pipelines — exact output format, strict step ordering, ranking, and calibrated refusal across six categories.", "launch_date": "2026-03-05T00:00:00.000Z", "status": "active", "status_evidence": "Newly released (arXiv paper submitted 5 March 2026); no refresh cycle or successor yet, and scores still separate frontier models. Worth tracking but with no independent replications published so far.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "FireBench: Evaluating Instruction Following in Enterprise and API-Driven LLM Applications", "url": "https://arxiv.org/abs/2603.04857", "venue": null }, "authors": [ { "name": "Zhang", "affiliation": null }, { "name": "Bei", "affiliation": null }, { "name": "Ravi", "affiliation": null }, { "name": "Garbacki", "affiliation": null } ], "maintainer": "Fireworks AI & Columbia University", "license": "CC-BY-4.0 (repository); paper distributed under CC BY-SA 4.0", "dataset": { "url": "https://github.com/FireBenchIF/FireBench", "size": 2470, "public_test_set": true, "gated": false }, "metric": { "primary": "composite instruction-following score", "scoring": "composite", "judge_model": null, "aggregation": "Normalized score across six capability categories (output format, ordered responses, item ranking, overconfidence, positive content, negative content); four categories are graded programmatically, and the two content-requirement categories are graded by a judge. Exact weighting between categories was not independently verified; category-level results are more interpretable than the single composite.", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": null, "pass_at": null, "tools": false, "harnesses": [ { "name": "Official FireBench repo (src/main.py)", "url": "https://github.com/FireBenchIF/FireBench", "task_flag": "python3 src/main.py" } ] }, "cost_estimate": { "examples": 2470, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No human baseline published. The categories map to concrete enterprise requirements, but the paper reports only model performance." }, "statistical_note": "2,470 samples across six categories → roughly ±2pt per category at ~50% accuracy (95% CI) if categories are evenly sized; the two content-requirement categories graded by judges carry extra variance from judge disagreement.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The benchmark is open-sourced at fire-bench.com and in the FireBenchIF/FireBench GitHub repo (CC-BY-4.0), and released a canary string discouraging inclusion in training corpora. However, the distributed data is a partial subset that is regenerated procedurally by the harness, and the paper's 1,300/2,470 remarked \"test instances\" are not pre-materialized. The categories target behavior (exact formats, ordering, refusal), which is harder to memorize than factual answers.", "risk": "low" }, "languages": [ "english" ], "leaderboards": [], "performance_timeline": [], "tags": [ "instruction-following", "enterprise", "api", "structured-output", "json", "ordered-responses", "refusal", "calibration" ], "last_updated": "2026-08-07T00:00:00.000Z", "changelog": [ { "date": "2026-08-07T00:00:00.000Z", "note": "Corrected frontmatter after source verification: launch_date set to the arXiv submission date 2026-03-05 (was 2026-03-01); public_test_set set to true, dataset.url and license filled from the public FireBenchIF/FireBench repo, and contamination/harness notes updated to match the repo. Timeline remains empty (no sourced per-model scores with status on this page yet) and per-category composite weights remain unpublished." }, { "date": "2026-08-07T00:00:00.000Z", "note": "Initial page. Timeline is empty (no sourced per-model scores yet); dataset.url and license left null pending public-release confirmation." } ] }, { "name": "FrontierCode", "slug": "frontiercode", "capability": "coding", "secondary_capabilities": [ "agentic-tool-use" ], "short_description": "Private maintainer-authored repository tasks graded for mergeability: correctness, tests, scope discipline, style, and code quality.", "launch_date": "2026-06-08T00:00:00.000Z", "status": "active", "status_evidence": "FrontierCode 1.1 remains difficult and newly launched; Cognition's official results show substantial headroom on the current Main and Extended sets.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "frontiercode-main", "frontiercode-extended" ] }, "paper": null, "authors": [ { "name": "Eric Lu et al.", "affiliation": "Cognition" }, { "name": "Open-source repository maintainers", "affiliation": null } ], "maintainer": "Cognition", "license": null, "dataset": { "url": "https://cognition.com/frontiercode", "size": 150, "public_test_set": false, "gated": true }, "metric": { "primary": "pass rate on FrontierCode 1.1 Main (%)", "scoring": "composite", "judge_model": null, "aggregation": "mean pass rate over five trials per task at maximum reasoning effort; a trial passes only if every blocker criterion is satisfied", "variants": [ { "name": "weighted mergeability score", "description": "Weighted aggregate of satisfied rubric items; failing any blocker makes that trial's score zero.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "one maintainer-written task brief per repository", "cot": "agentic", "temperature": null, "pass_at": null, "tools": true, "harnesses": [ { "name": "Cognition hosted FrontierCode evaluation", "url": "https://cognition.com/frontiercode", "task_flag": null } ] }, "cost_estimate": { "examples": 150, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No controlled human aggregate is published. Repository maintainers author the tasks and define mergeability criteria, but that provenance is not a measured human score." }, "statistical_note": "Main contains 100 tasks and Extended 150; results average five trials and select the best reasoning effort. Timeline dates are model launch dates; source_url points to Cognition's score table. Never mix versions, subsets, or pass rate with weighted score.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "Tasks and solutions remain private. FrontierCode 1.1 allows ordinary documentation search but prompts agents to avoid solution-bearing sources and programmatically zeroes runs that access them.", "risk": "low" }, "languages": [ "multilingual" ], "leaderboards": [ { "name": "FrontierCode official leaderboard", "url": "https://cognition.com/frontiercode", "type": "official" } ], "performance_timeline": [ { "model": "SWE-1.6 (FrontierCode 1.1 Main pass rate)", "vendor": "cognition", "date": "2026-04-07T00:00:00.000Z", "score": 9.4, "source": "vendor-reported", "source_url": "https://cognition.com/blog/swe-1-7" }, { "model": "Kimi K2.7 Code (FrontierCode 1.1 Main pass rate)", "vendor": "moonshot", "date": "2026-06-12T00:00:00.000Z", "score": 30.1, "source": "independent", "source_url": "https://cognition.com/blog/swe-1-7" }, { "model": "SWE-1.7 (FrontierCode 1.1 Main pass rate)", "vendor": "cognition", "date": "2026-07-08T00:00:00.000Z", "score": 42.3, "source": "vendor-reported", "source_url": "https://cognition.com/blog/swe-1-7" }, { "model": "GPT-5.5 (FrontierCode 1.1 Main pass rate)", "vendor": "openai", "date": "2026-04-23T00:00:00.000Z", "score": 43, "source": "independent", "source_url": "https://cognition.com/blog/swe-1-7" }, { "model": "Claude Opus 4.8 (FrontierCode 1.1 Main pass rate)", "vendor": "anthropic", "date": "2026-05-28T00:00:00.000Z", "score": 46.5, "source": "independent", "source_url": "https://cognition.com/blog/swe-1-7" } ], "tags": [ "software-engineering", "mergeability", "repositories", "agents", "code-quality", "private-test-set" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Changed FrontierCode score dates from the 2026-07-08 Cognition snapshot date to each model's launch date: SWE-1.6 2026-04-07, GPT-5.5 2026-04-23, Claude Opus 4.8 2026-05-28, Kimi K2.7 Code 2026-06-12, and SWE-1.7 2026-07-08." }, { "date": "2026-08-03T00:00:00.000Z", "note": "Added a fixed-axis FrontierCode 1.1 Main pass-rate snapshot from Cognition's 2026-07-08 table. Weighted score, Extended, and deprecated Diamond remain off-axis. Points share one table date because this is a cross-model snapshot, not a historical trend." } ] }, { "name": "FrontierMath", "slug": "frontiermath", "capability": "maths", "secondary_capabilities": [ "reasoning" ], "short_description": "Original, unpublished research-level math problems by expert mathematicians — mostly private to resist contamination. Frontier models solved under 2% at launch.", "launch_date": "2024-11-07T00:00:00.000Z", "status": "active", "status_evidence": "The strongest active case on the wiki. At launch (Nov 2024) state-of-the-art models solved UNDER 2% — a benchmark frontier systems almost entirely fail. Reasoning models with Python tools have since climbed (OpenAI announced o3 at 25.2% in Dec 2024, tools-on), but that figure is contested — Epoch's own independent reproductions of comparable models came in far lower (~11%). By the June 2026 v2 release Epoch added a harder Tier 4 precisely because Tiers 1–3 progress had become significant, yet the hardest tier remains far from solved. Current figures move fast and are tool/compute-dependent — consult Epoch's live dashboard; this page does not restate unverified third-party numbers. Saturation here would mean research-level, closed-form mathematics is largely automated.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "frontiermath-tier-4" ] }, "paper": { "title": "FrontierMath: A Benchmark for Evaluating Advanced Mathematical Reasoning in AI", "url": "https://arxiv.org/abs/2411.04872", "venue": "Epoch AI (preprint)" }, "authors": [ { "name": "Elliot Glazer", "affiliation": "Epoch AI" }, { "name": "Tamay Besiroglu", "affiliation": "Epoch AI" }, { "name": "+ a large collaborating group of research mathematicians", "affiliation": null } ], "maintainer": "Epoch AI (Glazer et al.), with problems authored by expert mathematicians; evaluations run by Epoch on the private set", "license": "proprietary (Epoch AI — private held-out set; a small set of sample problems is public)", "dataset": { "url": "https://epoch.ai/frontiermath", "size": 338, "public_test_set": false, "gated": true }, "metric": { "primary": "accuracy — % of problems solved, automated answer verification (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "per-tier accuracy on the PRIVATE set; answers are numerical / SymPy-checkable, so grading is automated equivalence, not human-judged. Reported per tier, not as one blended number", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "agentic", "temperature": null, "pass_at": 1, "tools": true, "harnesses": [ { "name": "Epoch AI (canonical — runs the private eval)", "url": "https://epoch.ai/frontiermath", "task_flag": null } ] }, "cost_estimate": { "examples": 338, "approx_usd_frontier_model": "High and highly variable: problems take specialists hours to days, so models emit very long agentic traces with Python execution; heavy-compute / high-sample runs (which drive the highest reported numbers) cost far more than a single pass" }, "human_baseline": { "score": null, "note": "No aggregate human baseline exists, and — as with HLE — none is well-defined in principle, only more so. These are research-level problems spanning many advanced fields; no single mathematician can solve across all of them. The meaningful human reference is per-problem: an expert in the relevant subfield solves it over hours to days. Fields Medalists who reviewed the problems underscored the point — Terence Tao said that in the near term 'the only way to solve them, short of having a real domain expert in the area, is to get a lucky overfit,' and predicted the hardest tier would 'resist AIs for several years.'" }, "statistical_note": "Scores are reported per tier, and the tiers are small (Tier 4 is 43 problems), so per-tier figures carry wide confidence intervals — a few problems move the percentage several points. Two deeper caveats dominate any FrontierMath number: (1) tools-on vs tools-off and heavy-compute vs single-pass swing results enormously (see Scoring methodology); (2) the set is private, so you are trusting Epoch's evaluation rather than reproducing it yourself. Always cite the tier, the tool/compute setting, and that it is an Epoch-run private-set number.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "The strongest contamination defense catalogued on the wiki — and the counter-model to AIME's yearly refresh and MMLU's fully-public exposure. The problems are ORIGINAL and UNPUBLISHED, and the benchmark is mostly PRIVATE by design: only a small sample set is public, and Epoch runs evaluations on a held-out set that is not released. The tradeoff is a different threat model: you cannot independently reproduce a private benchmark, so you must trust the evaluator (see Known issues). The privacy is also entangled with the funding history — OpenAI, which funded the dataset, had access to most problems, so Epoch established held-out sets the funder cannot see (53 solutions withheld in one version; 20 of 50 Tier 4 problems reserved) as a post-hoc safeguard.", "risk": "low" }, "languages": [ "english" ], "leaderboards": [ { "name": "Epoch AI (official / canonical tracker)", "url": "https://epoch.ai/frontiermath", "type": "official" } ], "performance_timeline": [ { "model": "best frontier model (Tiers 1–3, tools-on)", "vendor": "multiple", "date": "2024-11-07T00:00:00.000Z", "score": 2, "source": "independent", "source_url": "https://arxiv.org/abs/2411.04872" }, { "model": "o3-mini high (Epoch independent eval, Tiers 1–3, tools-on)", "vendor": "openai", "date": "2025-02-28T00:00:00.000Z", "score": 11, "source": "independent", "source_url": "https://epoch.ai/frontiermath" } ], "tags": [ "maths", "research-level", "private", "held-out", "contamination", "epoch", "disclosure" ], "last_updated": "2026-07-22T00:00:00.000Z", "changelog": [ { "date": "2026-07-22T00:00:00.000Z", "note": "Initial page. Fourth `maths` benchmark and the TOP of the GSM8K → MATH → AIME → FrontierMath headroom chain. Two firsts: first `public_test_set: false` (private held-out) and first `contamination.risk: low`. Sourcing was verified against Epoch AI and primary sources throughout; NO figures were taken from recollection or third-party aggregators. Timeline plots only Epoch-attributable numbers: launch best-frontier <2% (Epoch paper, arXiv:2411.04872) and Epoch's independent o3-mini-high eval ~11% (tools-on) — deliberately NOT OpenAI's announced 25.2%, which Epoch could not reproduce at that level (documented in Known issues). 2025–26 aggregator figures (reported in the 50–85% range for tools-on Tiers 1–3) were OMITTED as unverifiable against Epoch and given FrontierMath's inflation history. v2 (2026-06-12) expanded to 338 problems (Tiers 1–3: 295, Tier 4: 43). Funding-disclosure controversy documented neutrally in Known issues, cited to Epoch's own statement and TechCrunch. SCHEMA FLAG: this is the 4th benchmark (HLE, SimpleQA, AIME, FrontierMath) whose timeline points need structured tools/tier/sampling provenance — argued in the handoff summary." } ] }, { "name": "GLUE", "slug": "glue", "capability": "reasoning", "secondary_capabilities": [ "knowledge" ], "short_description": "Nine-task English NLU suite combining acceptability, sentiment, similarity, paraphrase, inference and coreference into one score.", "launch_date": "2018-04-20T00:00:00.000Z", "status": "saturated", "status_evidence": "RoBERTa reached 88.5 on the official nine-task leaderboard by 2019-07-25, above the conservative 87.1 non-expert human aggregate; SuperGLUE was introduced for more headroom.", "saturated_date": "2019-07-25T00:00:00.000Z", "successor": "superglue", "lineage": { "parent": null, "variants": [ "glue-diagnostic" ] }, "paper": { "title": "GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding", "url": "https://aclanthology.org/W18-5446/", "venue": "BlackboxNLP at EMNLP 2018" }, "authors": [ { "name": "Alex Wang", "affiliation": null }, { "name": "Amanpreet Singh", "affiliation": null }, { "name": "Julian Michael", "affiliation": null }, { "name": "Felix Hill", "affiliation": null }, { "name": "Omer Levy", "affiliation": null }, { "name": "Samuel R. Bowman", "affiliation": null } ], "maintainer": "NYU Machine Learning for Language", "license": null, "dataset": { "url": "https://huggingface.co/datasets/nyu-mll/glue", "size": null, "public_test_set": false, "gated": false }, "metric": { "primary": "GLUE Score (0-100)", "scoring": "composite", "judge_model": null, "aggregation": "Compute each task's official metric, average paired metrics within MRPC, QQP and STS-B and the matched/mismatched MNLI scores, then macro-average the nine task scores. The diagnostic set is reported separately and is not part of GLUE Score.", "variants": [ { "name": "CoLA", "description": "Binary linguistic acceptability; Matthews correlation coefficient.", "scoring": "exact-match", "judge_model": null }, { "name": "SST-2", "description": "Binary sentence sentiment; accuracy.", "scoring": "exact-match", "judge_model": null }, { "name": "MRPC", "description": "Binary paraphrase classification; accuracy and F1 are both reported and averaged for the suite aggregate.", "scoring": "composite", "judge_model": null }, { "name": "STS-B", "description": "Sentence-pair semantic similarity regression; Pearson and Spearman correlations are averaged.", "scoring": "composite", "judge_model": null }, { "name": "QQP", "description": "Duplicate-question classification; accuracy and F1 are both reported and averaged for the suite aggregate.", "scoring": "composite", "judge_model": null }, { "name": "MNLI", "description": "Three-way natural language inference; matched and mismatched test accuracies are averaged.", "scoring": "composite", "judge_model": null }, { "name": "QNLI", "description": "Binary question-sentence entailment classification; accuracy.", "scoring": "exact-match", "judge_model": null }, { "name": "RTE", "description": "Binary textual entailment across several small datasets; accuracy.", "scoring": "exact-match", "judge_model": null }, { "name": "WNLI", "description": "Winograd-derived binary NLI classification; accuracy. The split has a documented adversarial overlap quirk.", "scoring": "exact-match", "judge_model": null }, { "name": "GLUE diagnostic", "description": "Hand-built NLI examples grouped by linguistic phenomena; analyzed separately and excluded from the official aggregate.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "supervised fine-tuning per task; lm-evaluation-harness defaults to zero-shot validation evaluation", "cot": "direct", "temperature": null, "pass_at": 1, "tools": false, "harnesses": [ { "name": "GLUE official platform", "url": "https://gluebenchmark.com/", "task_flag": null }, { "name": "GLUE official baseline code", "url": "https://github.com/nyu-mll/GLUE-baselines", "task_flag": "all" }, { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness/tree/main/lm_eval/tasks/glue", "task_flag": "glue" } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": null }, "human_baseline": { "score": 87.1, "note": "Conservative non-expert crowd estimate from Nangia and Bowman (2019), based on brief instructions and 20 demonstrations per task; it is not an expert ceiling." }, "statistical_note": "A suite average hides very different sample sizes, metrics and uncertainty. Small tasks such as RTE and MRPC have high seed variance, while QQP and MNLI dominate training volume but still receive only one task's weight each.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "Four constituent tasks originally used private test data and official submissions hide labels, but all training/development data, several test sets, task sources and the diagnostic set are static and widely mirrored.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official GLUE leaderboard", "url": "https://gluebenchmark.com/leaderboard/", "type": "official" } ], "performance_timeline": [ { "model": "BERT Large", "vendor": "google", "date": "2018-10-11T00:00:00.000Z", "score": 80.5, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/1810.04805" }, { "model": "RoBERTa ensemble", "vendor": "facebook-ai", "date": "2019-07-25T00:00:00.000Z", "score": 88.5, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/1907.11692" }, { "model": "MT-DNN", "vendor": "microsoft", "date": "2019-02-25T00:00:00.000Z", "score": 82.7, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/1901.11504" }, { "model": "XLNet (ensemble)", "vendor": "google", "date": "2019-06-19T00:00:00.000Z", "score": 88.4, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/1906.08237" }, { "model": "ALBERT (ensemble)", "vendor": "google", "date": "2019-09-26T00:00:00.000Z", "score": 89.4, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/1909.11942" }, { "model": "T5 (ensemble)", "vendor": "google", "date": "2019-10-23T00:00:00.000Z", "score": 90.3, "source": "vendor-reported", "source_url": "https://gluebenchmark.com/leaderboard/" }, { "model": "ELECTRA-Large", "vendor": "google", "date": "2020-03-23T00:00:00.000Z", "score": 89.4, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2003.10555" }, { "model": "DeBERTa (TuringNLRv4)", "vendor": "microsoft", "date": "2021-01-06T00:00:00.000Z", "score": 90.8, "source": "vendor-reported", "source_url": "https://www.microsoft.com/en-us/research/blog/microsoft-deberta-surpasses-human-performance-on-the-superglue-benchmark/" } ], "tags": [ "suite", "nlu", "classification", "natural-language-inference", "sentiment", "semantic-similarity", "transfer-learning" ], "last_updated": "2026-07-25T00:00:00.000Z", "changelog": [ { "date": "2026-07-25T00:00:00.000Z", "note": "Initial page. Timeline is restricted to paper-reported official nine-task leaderboard aggregates; eight-task development averages and validation-only harness scores are excluded as non-comparable." }, { "date": "2026-07-25T00:00:00.000Z", "note": "Expanded performance timeline with MT-DNN, XLNet ensemble, ALBERT ensemble, T5 ensemble, ELECTRA-Large, and DeBERTa TuringNLRv4 scores." } ] }, { "name": "GPQA", "slug": "gpqa", "capability": "knowledge", "secondary_capabilities": [ "reasoning" ], "short_description": "PhD-written science questions so hard that skilled non-experts with Google score 34% — the 'Google-proof' exam, reported on its 198-question Diamond subset.", "launch_date": "2023-11-20T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "Frontier reasoning models reach the high 80s on GPQA Diamond (e.g. Grok 4 ~87%, mid-2025), against a label-noise-adjusted ceiling likely in the low 90s — the paper documents a few percent of questions with expert-identified errors. Unlike MMLU, meaningful headroom remains and top models are only now crossing the PhD-expert line, so it still discriminates; but the trajectory is one or two model generations from done. Judgment call — the first `nearing-saturation` benchmark on the wiki.", "saturated_date": null, "successor": "humanitys-last-exam", "lineage": { "parent": null, "variants": [ "gpqa-main", "gpqa-extended", "gpqa-diamond" ] }, "paper": { "title": "GPQA: A Graduate-Level Google-Proof Q&A Benchmark", "url": "https://arxiv.org/abs/2311.12022", "venue": "COLM 2024" }, "authors": [ { "name": "David Rein", "affiliation": "New York University" }, { "name": "Betty Li Hou", "affiliation": "New York University" }, { "name": "Julian Michael", "affiliation": "New York University / Cohere" }, { "name": "Samuel R. Bowman", "affiliation": "New York University / Anthropic, PBC" } ], "maintainer": "David Rein et al. (NYU, with Cohere and Anthropic-affiliated co-authors); repo github.com/idavidrein/gpqa", "license": "CC-BY-4.0", "dataset": { "url": "https://huggingface.co/datasets/Idavidrein/gpqa", "size": 198, "public_test_set": true, "gated": true }, "metric": { "primary": "accuracy on GPQA Diamond, 4-option multiple choice (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "mean accuracy over the 198 Diamond questions (the primary reported configuration; see the note on dataset.size)", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "cot", "temperature": null, "pass_at": 1, "tools": false, "harnesses": [ { "name": "GPQA (official)", "url": "https://github.com/idavidrein/gpqa", "task_flag": null }, { "name": "Inspect", "url": "https://inspect.aisi.org.uk", "task_flag": "gpqa_diamond" }, { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "gpqa_diamond_zeroshot" } ] }, "cost_estimate": { "examples": 198, "approx_usd_frontier_model": "2–15 on Diamond (198 items; reasoning-era models emit long chains of thought per question, which dominate token cost). The 448-question main or 546-question extended sets scale up proportionally." }, "human_baseline": { "score": 81.3, "note": "PhD-domain experts scored 81.3% on the Diamond subset (the primary configuration this page reports); non-expert validators with unrestricted web access scored only 22.1% on Diamond. On the full 448-question main set the headline figures are ~65% expert (74% after discounting mistakes the experts themselves flagged in retrospect) and ~34% non-expert. IMPORTANT nuance: OpenAI, evaluating o1, recruited its own PhD experts who scored only 69.7% on Diamond — so whether a model 'beats PhD experts' depends entirely on which expert pool you compare against. The gauge/timeline use the paper's 81.3% Diamond expert accuracy as the reference line, because GPQA's entire claim is expert-level difficulty; beating the 22–34% non-expert line is trivial and not the point." }, "statistical_note": "GPQA Diamond is only 198 questions. At ~80% accuracy that is a 95% confidence interval of roughly ±5.6 points (±7 near 50%). This is the single most important number on the page: a great many publicized 'Model A beats Model B by 2 points on GPQA Diamond' claims are statistically indistinguishable from noise. Treat any Diamond gap under ~6 points as unresolved, and prefer the larger main/extended sets when a comparison actually needs to be settled.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The dataset is public but gated: HuggingFace access requires accepting an agreement, and the authors ask users NOT to post questions or answers online 'to reduce the risk of leakage into foundation model training corpora.' This is a voluntary pledge, not a technical control — the same class of opt-in decontamination that failed for BIG-Bench's canary. The set is fully static with no refreshed variant, and Diamond questions have been discussed and reproduced online since 2023, which pushes the effective risk from medium toward high over time.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [ { "name": "Epoch AI (GPQA Diamond)", "url": "https://epoch.ai/benchmarks/gpqa-diamond", "type": "third-party" }, { "name": "Artificial Analysis (GPQA Diamond)", "url": "https://artificialanalysis.ai/evaluations/gpqa-diamond", "type": "third-party" } ], "performance_timeline": [ { "model": "GPT-4o", "vendor": "openai", "date": "2024-05-13T00:00:00.000Z", "score": 56.1, "source": "vendor-reported", "source_url": "https://openai.com/index/learning-to-reason-with-llms/" }, { "model": "o1", "vendor": "openai", "date": "2024-09-12T00:00:00.000Z", "score": 78, "source": "vendor-reported", "source_url": "https://openai.com/index/learning-to-reason-with-llms/" }, { "model": "Grok 4", "vendor": "xai", "date": "2025-07-09T00:00:00.000Z", "score": 87, "source": "independent", "source_url": "https://epoch.ai/benchmarks/gpqa-diamond" } ], "tags": [ "science", "graduate-level", "google-proof", "diamond", "multiple-choice", "contamination" ], "last_updated": "2026-07-21T00:00:00.000Z", "changelog": [ { "date": "2026-07-21T00:00:00.000Z", "note": "Initial page. First `nearing-saturation` status on the wiki. Timeline is on the GPQA Diamond axis (the primary reported configuration): GPT-4o 56.1 and o1 78.0 (vendor-reported, OpenAI o1 post) and Grok 4 87.0 (independent, Epoch AI). GPT-4's ~39% launch baseline is the paper's strongest baseline on the MAIN set, not Diamond, so it is described in prose but NOT plotted — mixing subsets on one axis would be dishonest, and no cleanly-sourced GPT-4 Diamond launch point was found. 2025–26 scores in the low-to-mid 90s are discussed in prose but not plotted (no single authoritative source verified per model). Human baseline stored as the paper's 81.3% Diamond expert accuracy; the main-set 65%/74% figures, the 22.1%/34% non-expert figures, and OpenAI's 69.7% recruited-expert figure are all in the note. dataset.size is 198 (Diamond) per the size-matches-primary-configuration convention documented in Task details." } ] }, { "name": "GSM-Hard", "slug": "gsm-hard", "capability": "maths", "secondary_capabilities": [ "reasoning" ], "short_description": "GSM8K test problems with unusually large numerical values, designed to separate reasoning from easy arithmetic.", "launch_date": "2023-01-17T00:00:00.000Z", "status": "deprecated", "status_evidence": "The public 1,319-item transformation remains a useful arithmetic stress test, but automatic number replacement creates implausible and sometimes invalid questions; it should not be used as a standalone reasoning benchmark.", "saturated_date": null, "successor": null, "lineage": { "parent": "gsm8k", "variants": [] }, "paper": { "title": "PAL: Program-aided Language Models", "url": "https://arxiv.org/abs/2211.10435", "venue": "ICML 2023" }, "authors": [ { "name": "Luyu Gao", "affiliation": null }, { "name": "Aman Madaan", "affiliation": null }, { "name": "Shuyan Zhou", "affiliation": null }, { "name": "Graham Neubig", "affiliation": "Carnegie Mellon University" } ], "maintainer": "Reasoning Machines", "license": "MIT (Hugging Face dataset card)", "dataset": { "url": "https://huggingface.co/datasets/reasoning-machines/gsm-hard", "size": 1319, "public_test_set": true, "gated": false }, "metric": { "primary": "final-answer accuracy (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "mean exact match over all 1,319 transformed GSM8K test questions after numeric answer extraction", "variants": [] }, "run_config": { "shots": "few-shot in the PAL paper; configuration varies in later use", "cot": "agentic", "temperature": 0, "pass_at": 1, "tools": true, "harnesses": [ { "name": "PAL reference repository", "url": "https://github.com/reasoning-machines/pal", "task_flag": null } ] }, "cost_estimate": { "examples": 1319, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No aggregate human baseline was published for the automatically transformed set." }, "statistical_note": "At 1,319 items, a 50% accuracy has an approximate 95% binomial interval of +/-2.7 points, but malformed transformations introduce non-sampling error that a confidence interval cannot capture.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "None beyond replacing GSM8K numbers; questions, generated programs, and targets are public.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [], "performance_timeline": [], "tags": [ "maths", "arithmetic", "robustness", "gsm8k", "program-aided" ], "last_updated": "2026-07-29T00:00:00.000Z", "changelog": [ { "date": "2026-07-29T00:00:00.000Z", "note": "Initial page. Marked deprecated for standalone capability claims because automatic substitutions can break realism and validity; no unsourced model scores were plotted." } ] }, { "name": "GSM8K", "slug": "gsm8k", "capability": "maths", "secondary_capabilities": [ "reasoning" ], "short_description": "8.5K grade-school math word problems (2–8 arithmetic steps) — the default elementary-math benchmark and the classic chain-of-thought demonstration.", "launch_date": "2021-10-27T00:00:00.000Z", "status": "saturated", "status_evidence": "Frontier models cleared ~95% by 2023–24 and the reasoning-model era pushed scores to ~97–99%, collapsing headroom. But saturation here is the sharpest 'saturated ≠ solved' case on the wiki: GSM-Symbolic (Apple, 2024) showed that changing only names/numbers, and especially adding one irrelevant clause (GSM-NoOp), drops accuracy heavily across models — so the SCORE is saturated while the CAPABILITY (robust arithmetic reasoning) is not. saturated_date 2024-01 marks the point frontier scores stopped discriminating; the discriminating signal moved to harder math (MATH → AIME → FrontierMath) and to robustness variants.", "saturated_date": "2024-01-01T00:00:00.000Z", "successor": null, "lineage": { "parent": null, "variants": [ "gsm-hard", "gsm-symbolic", "gsm-plus", "gsm8k-platinum" ] }, "paper": { "title": "Training Verifiers to Solve Math Word Problems", "url": "https://arxiv.org/abs/2110.14168", "venue": "OpenAI (preprint)" }, "authors": [ { "name": "Karl Cobbe", "affiliation": "OpenAI" }, { "name": "Vineet Kosaraju", "affiliation": "OpenAI" }, { "name": "John Schulman", "affiliation": "OpenAI" } ], "maintainer": "OpenAI (Cobbe et al.); repo openai/grade-school-math, dataset openai/gsm8k", "license": "MIT", "dataset": { "url": "https://huggingface.co/datasets/openai/gsm8k", "size": 1319, "public_test_set": true, "gated": false }, "metric": { "primary": "accuracy — exact match on the final numeric answer (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "mean over the 1,319 test problems; the answer after the '####' delimiter is compared, reasoning steps are not graded", "variants": [] }, "run_config": { "shots": "5-shot", "cot": "cot", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "gsm8k" }, { "name": "GSM8K (official)", "url": "https://github.com/openai/grade-school-math", "task_flag": null } ] }, "cost_estimate": { "examples": 1319, "approx_usd_frontier_model": "1–5 (1,319 short test problems; chain-of-thought traces are modest. maj@k voting, which some headline numbers use, multiplies cost by k)" }, "human_baseline": { "score": null, "note": "No measured human baseline. The paper states only an informal difficulty target: 'a bright middle school student should be able to solve every problem.' Because the problems are grade-school and were verified solvable, the implicit human ceiling is near 100% for a capable solver — but no human solve-rate was measured, so the score is null. Treat this like MMLU's estimated-baseline situation, only softer: a target description, not a number." }, "statistical_note": "1,319 test problems → ±~1.2pt (95% CI) near 95%, so at the saturated frontier ordinary model gaps are within noise. But the dominant caveat here is NOT sampling error — it is robustness: GSM-Symbolic shows the same model's accuracy swings by tens of points under trivial rewordings, so a single GSM8K number overstates reasoning reliability far more than its CI suggests. Also non-comparable across protocols (direct vs CoT vs maj@k).", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "None. Public since 2021, MIT-licensed, and mirrored across the web WITH worked natural-language solutions — ideal memorization fodder. Fully static, no refresh. Structurally the same exposure that undid MMLU, and a direct reason the GSM-Symbolic and GSM8K-Platinum variants exist: to test whether a score survives when the exact public items are perturbed or cleaned.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Papers with Code (GSM8K)", "url": "https://paperswithcode.com/sota/arithmetic-reasoning-on-gsm8k", "type": "third-party" } ], "performance_timeline": [ { "model": "GPT-3 175B (fine-tuned)", "vendor": "openai", "date": "2021-10-27T00:00:00.000Z", "score": 35, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2110.14168" }, { "model": "PaLM 540B (8-shot CoT)", "vendor": "google", "date": "2022-01-28T00:00:00.000Z", "score": 56.9, "source": "independent", "source_url": "https://arxiv.org/abs/2201.11903" }, { "model": "PaLM 540B (CoT + self-consistency)", "vendor": "google", "date": "2022-03-21T00:00:00.000Z", "score": 74.4, "source": "independent", "source_url": "https://arxiv.org/abs/2203.11171" }, { "model": "GPT-4 (5-shot CoT)", "vendor": "openai", "date": "2023-03-14T00:00:00.000Z", "score": 92, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2303.08774" } ], "tags": [ "maths", "word-problems", "grade-school", "chain-of-thought", "arithmetic", "contamination" ], "last_updated": "2026-07-22T00:00:00.000Z", "changelog": [ { "date": "2026-07-22T00:00:00.000Z", "note": "Initial page. First `maths` capability on the wiki. Timeline traces the canonical chain-of-thought arc on GSM8K accuracy: GPT-3 175B fine-tuned ~35% (paper Fig 5 baseline, vendor-reported, arXiv:2110.14168), PaLM 540B 8-shot CoT 56.9% (Wei et al., arXiv:2201.11903, independent), PaLM 540B CoT+self-consistency 74.4% (Wang et al., arXiv:2203.11171, independent), GPT-4 5-shot CoT 92.0% (GPT-4 report, arXiv:2303.08774, vendor-reported). The 2024+ reasoning-era plateau (~97–99%) is described in prose but NOT plotted — no single authoritative, dated, per-model source was confirmed, and GPT-4's 92% already reads as near-saturation. Protocols differ per point (fine-tuned / CoT / CoT+SC) — noted in Scoring methodology, not hidden. dataset.size 1,319 = the TEST set (the reported configuration), per the size-matches-primary convention; the 7,473-problem train split is in Task details." } ] }, { "name": "HellaSwag", "slug": "hellaswag", "capability": "reasoning", "secondary_capabilities": [ "knowledge" ], "short_description": "Choose the plausible continuation of an everyday event from one real ending and three adversarially filtered machine generations.", "launch_date": "2019-05-19T00:00:00.000Z", "status": "saturated", "status_evidence": "GPT-4 base reached 95.3% on a private holdout against the original 95.6% human reference; the official leaderboard closed submissions in 2024, and public validation data is contamination-prone.", "saturated_date": "2023-03-14T00:00:00.000Z", "successor": null, "lineage": { "parent": "swag", "variants": [] }, "paper": { "title": "HellaSwag: Can a Machine Really Finish Your Sentence?", "url": "https://arxiv.org/abs/1905.07830", "venue": "ACL 2019" }, "authors": [ { "name": "Rowan Zellers", "affiliation": null }, { "name": "Ari Holtzman", "affiliation": null }, { "name": "Yonatan Bisk", "affiliation": null }, { "name": "Ali Farhadi", "affiliation": null }, { "name": "Yejin Choi", "affiliation": null } ], "maintainer": "Rowan Zellers / University of Washington", "license": "MIT", "dataset": { "url": "https://huggingface.co/datasets/Rowan/hellaswag", "size": 10003, "public_test_set": false, "gated": false }, "metric": { "primary": "normalized accuracy (%)", "scoring": "log-likelihood", "judge_model": null, "aggregation": "mean over examples; the standard lm-evaluation-harness run reports validation-set acc_norm", "variants": [ { "name": "accuracy", "description": "Fraction of examples where the correct continuation receives the highest total log-likelihood.", "scoring": "log-likelihood", "judge_model": null }, { "name": "normalized accuracy", "description": "Fraction correct after dividing each continuation's log-likelihood by its token length; reported as acc_norm by lm-evaluation-harness.", "scoring": "log-likelihood", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "hellaswag" } ] }, "cost_estimate": { "examples": 10042, "approx_usd_frontier_model": null }, "human_baseline": { "score": 95.6, "note": "Overall test accuracy reported by the authors; random choice among four endings is 25%." }, "statistical_note": "On 10,003 independent test items, a score near 50% has an approximate 95% binomial margin of error of +/-1.0 point; near 95% it is about +/-0.4 point. Prompting and length normalization can move results by more than this.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "Official test labels were withheld and GPT-4 used a private holdout, but the standard harness evaluates the fully public 10,042-item validation split. Report the split and check training overlap.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official HellaSwag leaderboard (closed)", "url": "https://rowanzellers.com/hellaswag/", "type": "official" } ], "performance_timeline": [ { "model": "BERT-Large", "vendor": "google", "date": "2019-05-19T00:00:00.000Z", "score": 47.3, "source": "independent", "source_url": "https://rowanzellers.com/hellaswag/" }, { "model": "GPT-4 base (10-shot)", "vendor": "openai", "date": "2023-03-14T00:00:00.000Z", "score": 95.3, "source": "vendor-reported", "source_url": "https://cdn.openai.com/papers/gpt-4.pdf" }, { "model": "RoBERTa", "vendor": "facebook-ai", "date": "2019-07-25T00:00:00.000Z", "score": 85.2, "source": "independent", "source_url": "https://rowanzellers.com/hellaswag/" }, { "model": "GPT-3 175B (few-shot)", "vendor": "openai", "date": "2020-07-22T00:00:00.000Z", "score": 79.3, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2005.14165" }, { "model": "PaLM 540B", "vendor": "google", "date": "2022-04-01T00:00:00.000Z", "score": 83.4, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2204.02311" }, { "model": "LLaMA-65B", "vendor": "meta", "date": "2023-02-27T00:00:00.000Z", "score": 84.2, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2302.13971" }, { "model": "Gemini Ultra (10-shot decontaminated)", "vendor": "google", "date": "2023-12-06T00:00:00.000Z", "score": 87.8, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2312.11805" }, { "model": "Claude 3 Opus", "vendor": "anthropic", "date": "2024-03-04T00:00:00.000Z", "score": 95.4, "source": "vendor-reported", "source_url": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf" }, { "model": "Llama 3 70B", "vendor": "meta", "date": "2024-04-18T00:00:00.000Z", "score": 88, "source": "vendor-reported", "source_url": "https://ai.meta.com/blog/meta-llama-3/" }, { "model": "GPT-4o", "vendor": "openai", "date": "2024-08-01T00:00:00.000Z", "score": 95.3, "source": "vendor-reported", "source_url": "https://openai.com/index/gpt-4o-system-card/" } ], "tags": [ "commonsense", "completion", "multiple-choice", "adversarial-filtering", "activitynet", "wikihow" ], "last_updated": "2026-07-25T00:00:00.000Z", "changelog": [ { "date": "2026-07-25T00:00:00.000Z", "note": "Initial page. Marked saturated from GPT-4's human-level private-holdout result and the closure of the official leaderboard; documented validation-set contamination and scoring-protocol caveats." }, { "date": "2026-07-25T00:00:00.000Z", "note": "Expanded performance timeline with RoBERTa, GPT-3, PaLM, LLaMA, Gemini Ultra, Claude 3 Opus, Llama 3 70B, and GPT-4o scores." } ] }, { "name": "HumanEval", "slug": "humaneval", "capability": "coding", "secondary_capabilities": [], "short_description": "OpenAI's 164 hand-written Python function-synthesis tasks, graded by executing generated completions against unit tests.", "launch_date": "2021-07-07T00:00:00.000Z", "status": "saturated", "status_evidence": "Modern coding models routinely cluster near the ceiling on the small public set, while EvalPlus showed the original sparse tests accept materially incorrect programs.", "saturated_date": "2023-05-02T00:00:00.000Z", "successor": "humaneval-plus", "lineage": { "parent": null, "variants": [ "humaneval-plus", "multi-pl-e" ] }, "paper": { "title": "Evaluating Large Language Models Trained on Code", "url": "https://arxiv.org/abs/2107.03374", "venue": null }, "authors": [ { "name": "Mark Chen", "affiliation": "OpenAI" }, { "name": "Jerry Tworek", "affiliation": "OpenAI" }, { "name": "Heewoo Jun", "affiliation": "OpenAI" }, { "name": "Qiming Yuan", "affiliation": "OpenAI" } ], "maintainer": "OpenAI", "license": "MIT", "dataset": { "url": "https://github.com/openai/human-eval", "size": 164, "public_test_set": true, "gated": false }, "metric": { "primary": "pass@1 (%)", "scoring": "unit-tests", "judge_model": null, "aggregation": "unbiased pass@k estimator over 164 tasks; pass@1 is the headline single-sample probability", "variants": [ { "name": "pass@10", "description": "Estimated fraction of tasks solved by at least one of ten sampled completions.", "scoring": "unit-tests", "judge_model": null }, { "name": "pass@100", "description": "Estimated fraction of tasks solved by at least one of 100 sampled completions.", "scoring": "unit-tests", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0.2, "pass_at": 1, "tools": false, "harnesses": [ { "name": "OpenAI human-eval", "url": "https://github.com/openai/human-eval", "task_flag": "evaluate_functional_correctness" }, { "name": "EvalPlus", "url": "https://github.com/evalplus/evalplus", "task_flag": "humaneval" } ] }, "cost_estimate": { "examples": 164, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No controlled aggregate human baseline was published; the tasks were hand-written and reference solutions were authored by people, but that is not a timed pass@1 measurement." }, "statistical_note": "With 164 binary tasks, a score near 90% has a rough 95% binomial interval of about ±4.6 points before accounting for sampling and prompt variance.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The authors hand-wrote problems to avoid direct overlap with their training data, but prompts, tests, and canonical solutions have been public since 2021.", "risk": "high" }, "languages": [ "python" ], "leaderboards": [ { "name": "EvalPlus leaderboard", "url": "https://evalplus.github.io/leaderboard.html", "type": "third-party" } ], "performance_timeline": [ { "model": "Codex-12B (original paper, 164-task HumanEval pass@1 estimator)", "vendor": "openai", "date": "2021-07-07T00:00:00.000Z", "score": 28.81, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2107.03374" }, { "model": "GPT-4 (EvalPlus Table 3, greedy, HumanEval base pass@1)", "vendor": "openai", "date": "2023-05-02T00:00:00.000Z", "score": 88.4, "source": "independent", "source_url": "https://arxiv.org/abs/2305.01210" }, { "model": "Qwen2.5-Coder-32B-Instruct (EvalPlus, greedy, HumanEval base pass@1)", "vendor": "alibaba", "date": "2024-11-12T00:00:00.000Z", "score": 92.7, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2409.12186" } ], "tags": [ "python", "function-synthesis", "unit-tests", "pass-at-k", "contamination" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Initial page. Timeline uses the 164-task HumanEval base pass@1 axis: Codex-12B from the original paper, GPT-4 greedy from EvalPlus Table 3, and Qwen2.5-Coder-32B-Instruct greedy from its technical report; model labels expose protocol differences." } ] }, { "name": "HumanEval+", "slug": "humaneval-plus", "capability": "coding", "secondary_capabilities": [], "short_description": "HumanEval's 164 Python tasks re-evaluated with roughly 80 times more tests to expose plausible but incorrect programs.", "launch_date": "2023-05-02T00:00:00.000Z", "status": "saturated", "status_evidence": "The official EvalPlus leaderboard reports o1-preview and o1-mini at 89.0% HumanEval+ pass@1; with only 164 fully public prompts, remaining score gaps are too small and exposure-prone for a reliable frontier ranking.", "saturated_date": "2024-09-12T00:00:00.000Z", "successor": null, "lineage": { "parent": "humaneval", "variants": [ "humaneval-plus-mini" ] }, "paper": { "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation", "url": "https://arxiv.org/abs/2305.01210", "venue": "NeurIPS 2023" }, "authors": [ { "name": "Jiawei Liu", "affiliation": "University of Illinois Urbana-Champaign" }, { "name": "Chunqiu Steven Xia", "affiliation": "University of Illinois Urbana-Champaign" }, { "name": "Yuyao Wang", "affiliation": "Nanjing University" }, { "name": "Lingming Zhang", "affiliation": "University of Illinois Urbana-Champaign" } ], "maintainer": "EvalPlus team", "license": "Apache-2.0 (EvalPlus repository)", "dataset": { "url": "https://huggingface.co/datasets/evalplus/humanevalplus", "size": 164, "public_test_set": true, "gated": false }, "metric": { "primary": "HumanEval+ pass@1 (%)", "scoring": "unit-tests", "judge_model": null, "aggregation": "fraction of 164 tasks for which one generated completion passes all base and plus tests", "variants": [ { "name": "HumanEval base pass@1", "description": "The same completions graded only against the original HumanEval tests, useful for measuring the test-strength gap.", "scoring": "unit-tests", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "EvalPlus", "url": "https://github.com/evalplus/evalplus", "task_flag": "humaneval" } ] }, "cost_estimate": { "examples": 164, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No controlled aggregate human pass@1 baseline has been published for HumanEval+; human-written canonical solutions are test oracles, not a timed baseline." }, "statistical_note": "Only 164 task-level outcomes are averaged; near 90%, the rough 95% binomial interval is about ±4.6 points, separate from evaluator-version and generation variance.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Extra tests make shallow overfitting harder, but the prompts, canonical solutions, and generated tests are public and versioned rather than continuously refreshed.", "risk": "high" }, "languages": [ "python" ], "leaderboards": [ { "name": "Official EvalPlus leaderboard", "url": "https://evalplus.github.io/leaderboard.html", "type": "official" } ], "performance_timeline": [ { "model": "o1-preview (September 2024)", "vendor": "openai", "date": "2024-09-12T00:00:00.000Z", "score": 89, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "o1-mini (September 2024)", "vendor": "openai", "date": "2024-09-12T00:00:00.000Z", "score": 89, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "Qwen2.5-Coder-32B-Instruct", "vendor": "alibaba", "date": "2024-11-12T00:00:00.000Z", "score": 87.2, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "GPT-4o (August 2024)", "vendor": "openai", "date": "2024-08-06T00:00:00.000Z", "score": 87.2, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "DeepSeek-V3 (November 2024)", "vendor": "deepseek", "date": "2024-11-01T00:00:00.000Z", "score": 86.6, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "GPT-4-Turbo (April 2024)", "vendor": "openai", "date": "2024-04-09T00:00:00.000Z", "score": 86.6, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "DeepSeek-V2.5 (November 2024)", "vendor": "deepseek", "date": "2024-11-01T00:00:00.000Z", "score": 83.5, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "GPT-4o Mini (July 2024)", "vendor": "openai", "date": "2024-07-18T00:00:00.000Z", "score": 83.5, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "DeepSeek-Coder-V2-Instruct", "vendor": "deepseek", "date": "2024-06-17T00:00:00.000Z", "score": 82.3, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "Claude 3.5 Sonnet (June 2024)", "vendor": "anthropic", "date": "2024-06-20T00:00:00.000Z", "score": 81.7, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "GPT-4-Turbo (November 2023)", "vendor": "openai", "date": "2023-11-06T00:00:00.000Z", "score": 81.7, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "Grok Beta", "vendor": "xai", "date": "2024-11-04T00:00:00.000Z", "score": 80.5, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "Gemini 1.5 Pro 002", "vendor": "google", "date": "2024-09-24T00:00:00.000Z", "score": 79.3, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "GPT-4 (May 2023)", "vendor": "openai", "date": "2023-05-02T00:00:00.000Z", "score": 79.3, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" }, { "model": "CodeQwen1.5-7B-Chat", "vendor": "alibaba", "date": "2024-04-18T00:00:00.000Z", "score": 78.7, "source": "independent", "source_url": "https://evalplus.github.io/leaderboard.html" } ], "tags": [ "python", "function-synthesis", "unit-tests", "fuzzing", "pass-at-k", "contamination" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Expanded the timeline to all 15 official EvalPlus leaderboard rows supplied by the user, from CodeQwen1.5-7B-Chat at 78.7% through o1-preview and o1-mini at 89.0%; tied scores and model-version labels are preserved." }, { "date": "2026-08-03T00:00:00.000Z", "note": "Reclassified from nearing saturation to saturated at the user's direction. Added the official EvalPlus leaderboard's September 2024 o1-preview and o1-mini results of 89.0% HumanEval+ pass@1 as threshold evidence." }, { "date": "2026-08-03T00:00:00.000Z", "note": "Initial page. Timeline is a single apples-to-apples snapshot from Qwen2.5-Coder Table 16: greedy EvalPlus HumanEval+ pass@1 for the 7B (84.1) and 32B (87.2) Instruct models. Earlier GPT-4 results use an older HumanEval+ revision and remain prose-only." } ] }, { "name": "Humanity's Last Exam", "slug": "humanitys-last-exam", "capability": "knowledge", "secondary_capabilities": [ "reasoning" ], "short_description": "~2,500 expert-written questions across 100+ subjects, each filtered to stump frontier models — designed to be the final closed-ended academic benchmark.", "launch_date": "2025-01-24T00:00:00.000Z", "status": "active", "status_evidence": "The clearest active benchmark on the wiki. Launch scores were single digits (GPT-4o 3.3%, o1 9.1%, DeepSeek-R1 9.4%, no tools); the current no-tools frontier is ~38% (Gemini 3 Pro, Nov 2025) — a steep climb but a long way from solved. Given the 'last exam' thesis, saturation here would mean closed-ended academic testing has run out of headroom entirely, so 'what replaces it' points toward open-ended, agentic, and interactive evaluation rather than a harder Q&A set. Note: tool-augmented and agent scores run much higher and are tracked separately (see Timeline).", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "Humanity's Last Exam", "url": "https://arxiv.org/abs/2501.14249", "venue": "CAIS & Scale AI (preprint)" }, "authors": [ { "name": "Long Phan", "affiliation": "Center for AI Safety" }, { "name": "Alice Gatti", "affiliation": "Center for AI Safety" }, { "name": "Dan Hendrycks", "affiliation": "Center for AI Safety" } ], "maintainer": "Center for AI Safety (CAIS) & Scale AI; lastexam.ai, repo centerforaisafety/hle", "license": "MIT", "dataset": { "url": "https://huggingface.co/datasets/cais/hle", "size": 2500, "public_test_set": true, "gated": true }, "metric": { "primary": "accuracy, no tools (%)", "scoring": "llm-judge", "judge_model": "GPT-4o (gpt-4o-2024-08-06, structured decoding) checks whether a model's answer is equivalent to the reference for the 80% short-answer questions. Judge model/version drift makes scores across grader versions not strictly comparable — pin the grader.", "aggregation": "mean accuracy over ~2,500 questions; the headline configuration is closed-book (no tools, no browsing)", "variants": [ { "name": "RMS calibration error", "description": "A second headline number, derived from the same graded correctness plus each answer's stated confidence: the root-mean-square gap between confidence and correctness (implementation from Hendrycks et al.). Launch models were ~80–93% — i.e. highly confident while almost always wrong. Not a separate grading protocol but a distinct measurement of overconfidence; reported alongside accuracy, never instead of it.", "scoring": "llm-judge", "judge_model": "same GPT-4o judge (correctness component); confidence is model-reported" } ] }, "run_config": { "shots": "zero-shot", "cot": "cot", "temperature": null, "pass_at": 1, "tools": false, "harnesses": [ { "name": "HLE (official)", "url": "https://github.com/centerforaisafety/hle", "task_flag": null }, { "name": "Inspect", "url": "https://inspect.aisi.org.uk", "task_flag": "hle" } ] }, "cost_estimate": { "examples": 2500, "approx_usd_frontier_model": "10–50 no-tools (~2,500 hard questions elicit long reasoning traces, plus a GPT-4o judge call each). Tool-augmented / agentic runs cost far more — browsing and multi-step search can multiply token use by 10x or more." }, "human_baseline": { "score": null, "note": "No human baseline — and here the concept itself breaks down. HLE spans 100+ expert subjects from classics to rocket engineering; no single human could sit the whole exam, so an aggregate 'human score' is undefined in principle, not merely unmeasured. This differs from SimpleQA, where a human COULD take the test but the score would mostly measure Googling: HLE is superhuman in BREADTH by construction. The only meaningful human reference is per-question — each was solved by the domain expert who wrote it — which is exactly why the benchmark exists: to test whether one model can match the union of many human specialists." }, "statistical_note": "~2,500 questions give a workable aggregate CI (±~2pt at 40%), but two non-sampling caveats dominate. First, scores move FAST — a published number is often stale within months (single digits to ~38% in under a year), so always read HLE figures with their date and this page's last_updated. Second, the number is meaningless without its CONFIGURATION: no-tools vs tool-augmented vs multi-agent 'heavy' modes differ by tens of points on the same model (see Timeline). Read accuracy beside calibration error, and never compare across tool settings.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The strongest mitigation design catalogued on the wiki so far: alongside the public ~2,500-question set, CAIS/Scale hold a PRIVATE set of questions used to detect benchmark-specific overfitting by comparing a model's public vs private performance — a static-but-audited design. The public set is gated on HuggingFace (accept an agreement; users are asked not to re-share or re-upload). This does not make it leak-proof — the public questions are discussed online and the failure-filter selected for 2024-frontier blind spots — but the private audit catches the crudest gaming, which is why the risk is medium rather than high despite a fully static public set.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [ { "name": "Humanity's Last Exam (official, CAIS)", "url": "https://lastexam.ai", "type": "official" }, { "name": "Scale AI / SEAL", "url": "https://scale.com/leaderboard", "type": "third-party" } ], "performance_timeline": [ { "model": "GPT-4o", "vendor": "openai", "date": "2025-01-24T00:00:00.000Z", "score": 3.3, "source": "independent", "source_url": "https://arxiv.org/abs/2501.14249" }, { "model": "Claude 3.5 Sonnet", "vendor": "anthropic", "date": "2025-01-24T00:00:00.000Z", "score": 4.3, "source": "independent", "source_url": "https://arxiv.org/abs/2501.14249" }, { "model": "o1", "vendor": "openai", "date": "2025-01-24T00:00:00.000Z", "score": 9.1, "source": "independent", "source_url": "https://arxiv.org/abs/2501.14249" }, { "model": "DeepSeek-R1", "vendor": "deepseek", "date": "2025-01-24T00:00:00.000Z", "score": 9.4, "source": "independent", "source_url": "https://arxiv.org/abs/2501.14249" }, { "model": "Grok 4", "vendor": "xai", "date": "2025-07-09T00:00:00.000Z", "score": 24.5, "source": "independent", "source_url": "https://lastexam.ai" }, { "model": "GPT-5", "vendor": "openai", "date": "2025-08-07T00:00:00.000Z", "score": 25.3, "source": "independent", "source_url": "https://lastexam.ai" }, { "model": "Gemini 3 Pro", "vendor": "google", "date": "2025-11-18T00:00:00.000Z", "score": 38.3, "source": "independent", "source_url": "https://lastexam.ai" } ], "tags": [ "frontier", "closed-ended", "multi-subject", "calibration", "llm-judge", "private-holdout", "contamination" ], "last_updated": "2026-07-21T00:00:00.000Z", "changelog": [ { "date": "2026-07-21T00:00:00.000Z", "note": "Initial page. Successor target of GPQA (resolves GPQA's successor banner to a live link). Timeline policy: NO-TOOLS scores only — launch points from the paper Table 1 (GPT-4o 3.3, Claude 3.5 Sonnet 4.3, o1 9.1, DeepSeek-R1 9.4; all independent, arXiv:2501.14249) and later frontier points from the CAIS leaderboard (Grok 4 24.5 2025-07, GPT-5 25.3 2025-08, Gemini 3 Pro 38.3 2025-11; lastexam.ai). Tool-augmented / agentic scores (much higher) are deliberately EXCLUDED from the timeline and discussed in prose — see the note flagging a future per-point tools flag. Current no-tools SOTA is ~38% (NOT the ~50% sometimes recalled) per the live dashboard. dataset.size 2,500 = the current public set (refined down from the paper's 3,000 after post-launch answer-key review), per the size-matches-primary convention. First partially-private benchmark: public set gated on HF (cais/hle, MIT), private held-out set with CAIS/Scale. Human baseline null and, uniquely, undefined in principle." } ] }, { "name": "IFEval", "slug": "ifeval", "capability": "instruction-following", "secondary_capabilities": [], "short_description": "~500 prompts, each carrying an explicit, machine-checkable constraint (format, content, or style) that a response must respect.", "launch_date": "2023-11-14T00:00:00.000Z", "status": "deprecated", "status_evidence": "Deprecated in this wiki for frontier instruction-following comparison: it remains a useful cheap regression check, but its simple verifiable prompts have become a floor and newer suites cover multi-constraint, multi-turn, and agentic instruction following more directly.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "multi-if", "m-ifeval" ] }, "paper": { "title": "Instruction-Following Evaluation for Large Language Models", "url": "https://arxiv.org/abs/2311.07911", "venue": null }, "authors": [ { "name": "Jeffrey Zhou", "affiliation": "Yale University" }, { "name": "Tianjian Lu", "affiliation": "Google Research" }, { "name": "Swaroop Mishra", "affiliation": "Google Research" }, { "name": "Siddhartha Brahma", "affiliation": "Google Research" }, { "name": "Sujoy Basu", "affiliation": "Google Research" }, { "name": "Yi Luan", "affiliation": "Google Research" }, { "name": "Denny Zhou", "affiliation": "Google Research" }, { "name": "Le Hou", "affiliation": "Google Research" } ], "maintainer": "Google Research (google-research/instruction_following_eval)", "license": "Apache-2.0 (code); data released under its own terms", "dataset": { "url": "https://github.com/google-research/google-research/tree/master/instruction_following_eval", "size": 541, "public_test_set": true, "gated": false }, "metric": { "primary": "prompt- and instruction-level accuracy, strict and loose (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "per prompt (all constraints satisfied) and per instruction (each constraint graded separately), averaged over the ~500-prompt set; strict requires full compliance, loose tolerates leniently matched constraints", "variants": [ { "name": "strict prompt / instruction accuracy", "description": "A prompt counts as correct only if every verifiable instruction in it passes its check. The stricter, more commonly headline number.", "scoring": "exact-match", "judge_model": null }, { "name": "loose prompt / instruction accuracy", "description": "Lenient variant that accepts keyword-level matches and permits some instruction types to be skipped without failing the whole response.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "ifeval" }, { "name": "Official repo / Google instructional prompts", "url": "https://github.com/google-research/google-research/tree/master/instruction_following_eval", "task_flag": null } ] }, "cost_estimate": { "examples": 541, "approx_usd_frontier_model": "sub-$10 with a frontier API; short prompts and cheap rule-based grading, no LLM judge pass" }, "human_baseline": { "score": null, "note": "No protocol-comparable human score was published. The constraints are deliberately trivial for a human to meet, so a nominal human baseline would sit near 100% and be only weakly informative." }, "statistical_note": "541 prompts place roughly a ±4pt 95% interval on a 50% prompt-level score; per-instruction splits are noisier but finer-grained.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The paper includes an explicit prompt-level decontamination layer that filters candidates appearing in training data. The set is static and public, so memorization remains possible for models trained after release.", "risk": "low" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official repo (data and graders)", "url": "https://github.com/google-research/google-research/tree/master/instruction_following_eval", "type": "official" }, { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness/tree/main/lm_eval/tasks/ifeval", "type": "third-party" } ], "performance_timeline": [], "tags": [ "instruction-following", "verifiable-instructions", "format-compliance", "baseline" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2026-08-12T00:00:00.000Z", "note": "Changed status from active to deprecated. saturated_date remains null because deprecated pages are not required to declare a saturation crossing date." }, { "date": "2026-08-07T00:00:00.000Z", "note": "Initial page. performance_timeline is empty by design: the set is trivially hard for modern models, scores saturate in the high 80s–90s, and per-harness / per-variant reporting inconsistent, so no single comparable sourced series is maintained here. The scoring gap is noted rather than filled." } ] }, { "name": "Indirect Prompt Injection Arena", "slug": "indirect-prompt-injection", "capability": "safety-alignment", "secondary_capabilities": [ "agentic-tool-use", "coding", "multimodal" ], "short_description": "Forty-one agent scenarios derived from a 272,000-attempt public competition on concealed indirect prompt injection.", "launch_date": "2026-03-16T00:00:00.000Z", "status": "active", "status_evidence": "This is a current, agent-level security benchmark with tool, coding, and browser scenarios; every tested frontier model was vulnerable in the 2026 competition.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "ipi-arena-competition", "ipi-arena-open-source" ] }, "paper": { "title": "How Vulnerable Are AI Agents to Indirect Prompt Injections? Insights from a Large-Scale Public Competition", "url": "https://arxiv.org/html/2603.15714v1", "venue": null }, "authors": [ { "name": "Mateusz Dziemian", "affiliation": null }, { "name": "Maxwell Lin", "affiliation": null }, { "name": "Xiaohan Fu", "affiliation": null }, { "name": "Micha Nowak", "affiliation": null }, { "name": "Nick Winter", "affiliation": null }, { "name": "Eliot Jones", "affiliation": null }, { "name": "Andy Zou", "affiliation": null }, { "name": "Lama Ahmad", "affiliation": null }, { "name": "Kamalika Chaudhuri", "affiliation": null }, { "name": "Sahana Chennabasappa", "affiliation": null }, { "name": "Xander Davies", "affiliation": null }, { "name": "Lauren Deason", "affiliation": null }, { "name": "Benjamin L. Edelman", "affiliation": null }, { "name": "Tanner Emek", "affiliation": null }, { "name": "Ivan Evtimov", "affiliation": null }, { "name": "Jim Gust", "affiliation": null }, { "name": "Maia Hamin", "affiliation": null }, { "name": "Kat He", "affiliation": null }, { "name": "Klaudia Krawiecka", "affiliation": null }, { "name": "Riccardo Patana", "affiliation": null }, { "name": "Neil Perry", "affiliation": null }, { "name": "Troy Peterson", "affiliation": null }, { "name": "Xiangyu Qi", "affiliation": null }, { "name": "Javier Rando", "affiliation": null }, { "name": "Zifan Wang", "affiliation": null }, { "name": "Zihan Wang", "affiliation": null }, { "name": "Spencer Whitman", "affiliation": null }, { "name": "Eric Winsor", "affiliation": null }, { "name": "Arman Zharmagambetov", "affiliation": null }, { "name": "Matt Fredrikson", "affiliation": null }, { "name": "Zico Kolter", "affiliation": null } ], "maintainer": "Gray Swan AI and IPI Arena collaborators", "license": "MIT open-source benchmark", "dataset": { "url": "https://github.com/GraySwanAI/ipi_arena_os", "size": 41, "public_test_set": true, "gated": false }, "metric": { "primary": "concealed attack success rate (%) - lower is safer", "scoring": "composite", "judge_model": null, "aggregation": "an attack succeeds only when scenario-specific harmful-action criteria pass and the compromise remains concealed under tool-call and LLM-judge checks", "variants": [ { "name": "tool-call success", "description": "Deterministic check that the target agent called a prohibited or attacker-specified tool with the required arguments.", "scoring": "exact-match", "judge_model": null }, { "name": "concealment and task-completion criteria", "description": "Scenario-specific LLM-judge scores check whether the attack remained hidden and the visible user task still appeared complete.", "scoring": "llm-judge", "judge_model": "Configurable in the open-source harness" } ] }, "run_config": { "shots": "single-turn attack with a multi-step agent trajectory", "cot": "agentic", "temperature": null, "pass_at": null, "tools": true, "harnesses": [ { "name": "IPI Arena open-source benchmark", "url": "https://github.com/GraySwanAI/ipi_arena_os", "task_flag": "ipi-arena-bench run --config examples/hf_attacks.yaml" } ] }, "cost_estimate": { "examples": 41, "approx_usd_frontier_model": "Potentially high: each attack can trigger target, judge, WorldSim, and tool-loop calls." }, "human_baseline": { "score": null, "note": "No human agent baseline is defined. The source evidence is adversarial competition success against model-based agents." }, "statistical_note": "The public competition had 464 participants, 272,000 attack attempts, 8,648 successful attacks, 13 frontier models, and 41 scenarios. Timeline points use overall competition ASR, not curated transfer ASR; lower is safer. Timeline dates are model public-launch dates, not competition or evaluation dates.", "contamination": { "test_set_public": true, "refresh_cycle": "continuous", "mitigation": "The open release includes 41 behaviors and 95 attacks across 28 behaviors, while production evaluation should add private scenarios, rotate untrusted-content locations, and keep tool side effects sandboxed.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [], "performance_timeline": [ { "model": "Gemini 2.5 Pro (competition ASR)", "vendor": "google", "date": "2025-03-25T00:00:00.000Z", "score": 8.5, "source": "independent", "source_url": "https://arxiv.org/html/2603.15714v1" }, { "model": "Kimi K2 (competition ASR)", "vendor": "moonshot-ai", "date": "2025-07-11T00:00:00.000Z", "score": 4.8, "source": "independent", "source_url": "https://arxiv.org/html/2603.15714v1" }, { "model": "Claude Sonnet 4.5 (competition ASR)", "vendor": "anthropic", "date": "2025-09-29T00:00:00.000Z", "score": 1, "source": "independent", "source_url": "https://arxiv.org/html/2603.15714v1" }, { "model": "Claude Haiku 4.5 (competition ASR)", "vendor": "anthropic", "date": "2025-10-15T00:00:00.000Z", "score": 1.3, "source": "independent", "source_url": "https://arxiv.org/html/2603.15714v1" }, { "model": "Claude Opus 4.5 (competition ASR)", "vendor": "anthropic", "date": "2025-11-24T00:00:00.000Z", "score": 0.5, "source": "independent", "source_url": "https://arxiv.org/html/2603.15714v1" } ], "tags": [ "prompt-injection", "agents", "tool-use", "coding-agents", "computer-use", "red-teaming" ], "last_updated": "2026-09-14T00:00:00.000Z", "changelog": [ { "date": "2026-09-14T00:00:00.000Z", "note": "Added paper-reported competition ASRs using verified model public-launch dates for the timeline x-axis." }, { "date": "2026-09-14T00:00:00.000Z", "note": "Initial page based on arXiv v1, the public competition results, and the official open-source 41-scenario harness." } ] }, { "name": "InfiniteBench", "slug": "infinitebench", "capability": "long-context", "secondary_capabilities": [ "coding", "maths", "reasoning" ], "short_description": "100k+ token benchmark mixing realistic and synthetic tasks across English, Chinese, code, math, and retrieval.", "launch_date": "2024-02-21T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "InfiniteBench remains valuable for 100k+ token stress testing, but its public static split and newer complementary suites make it better classified as nearing-saturation than as the freshest active frontier signal.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "InfinityBench: Extending Long Context Evaluation Beyond 100K Tokens", "url": "https://arxiv.org/abs/2402.13718", "venue": "ACL 2024" }, "authors": [ { "name": "Xinrong Zhang", "affiliation": null }, { "name": "Yingfa Chen", "affiliation": null }, { "name": "Shengding Hu", "affiliation": null }, { "name": "Zihang Xu", "affiliation": null }, { "name": "Junhao Chen", "affiliation": null }, { "name": "Moo Khai Hao", "affiliation": null }, { "name": "Xu Han", "affiliation": null }, { "name": "Zhen Leng Thai", "affiliation": null }, { "name": "Shuo Wang", "affiliation": null }, { "name": "Zhiyuan Liu", "affiliation": null }, { "name": "Maosong Sun", "affiliation": null } ], "maintainer": "OpenBMB", "license": "MIT", "dataset": { "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "size": 3932, "public_test_set": true, "gated": false }, "metric": { "primary": "task-specific accuracy / ROUGE aggregate", "scoring": "composite", "judge_model": null, "aggregation": "per-task metrics over 12 tasks; most tasks use accuracy, English/Chinese QA use ROUGE F1, and English summarization uses ROUGE-Lsum", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official InfiniteBench repository", "url": "https://github.com/OpenBMB/InfiniteBench", "task_flag": "python src/eval_gpt4.py --task " } ] }, "cost_estimate": { "examples": 3932, "approx_usd_frontier_model": "high; many examples exceed 100k input tokens" }, "human_baseline": { "score": null, "note": "No single protocol-comparable human baseline is recorded." }, "statistical_note": "Input lengths and metrics vary sharply by task; quote per-task scores rather than a single aggregate. The visual graph on this page deliberately plots only the official InfiniteBench paper Table 3 Average row for the four baseline models. It does not plot the individual subset rows.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The data and construction scripts are public. Synthetic tasks can be regenerated, but the released benchmark split is static.", "risk": "high" }, "languages": [ "english", "chinese", "code" ], "leaderboards": [ { "name": "Official InfiniteBench repository", "url": "https://github.com/OpenBMB/InfiniteBench", "type": "official" } ], "performance_timeline": [ { "model": "GPT-4 (paper Table 3 Average)", "vendor": "openai", "date": "2024-02-21T00:00:00.000Z", "score": 45.63, "source": "independent", "source_url": "https://arxiv.org/abs/2402.13718" }, { "model": "YaRN-Mistral (paper Table 3 Average)", "vendor": "independent", "date": "2024-02-21T00:00:00.000Z", "score": 19.96, "source": "independent", "source_url": "https://arxiv.org/abs/2402.13718" }, { "model": "Kimi-Chat (paper Table 3 Average)", "vendor": "moonshot-ai", "date": "2024-02-21T00:00:00.000Z", "score": 34.73, "source": "independent", "source_url": "https://arxiv.org/abs/2402.13718" }, { "model": "Claude 2 (paper Table 3 Average)", "vendor": "anthropic", "date": "2024-02-21T00:00:00.000Z", "score": 37.06, "source": "independent", "source_url": "https://arxiv.org/abs/2402.13718" } ], "tags": [ "long-context", "hundred-k", "suite", "retrieval", "code", "math" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2024-02-21T00:00:00.000Z", "note": "Added official paper Table 3 Average scores to the visual graph and clarified that only the Average row is plotted, not the individual subset rows." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Initial long-context page. Dataset size computed from the official task-composition table example counts." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Moved InfiniteBench from active to nearing-saturation at user request; it remains useful for 100k+ stress testing but should not be the main active long-context signal." } ] }, { "name": "JailbreakBench", "slug": "jailbreakbench", "capability": "safety-alignment", "secondary_capabilities": [ "instruction-following" ], "short_description": "An open framework, behavior set, artifact repository, and leaderboard for comparing jailbreak attacks and defenses.", "launch_date": "2024-03-28T00:00:00.000Z", "status": "active", "status_evidence": "JailbreakBench remains a useful standardized red-teaming framework because it fixes behaviors, threat models, templates, judges, and artifact formats, while accepting evolving attacks and defenses.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "jbb-behaviors-harmful", "jbb-behaviors-benign", "jailbreakbench-artifacts" ] }, "paper": { "title": "JailbreakBench: An Open Robustness Benchmark for Jailbreaking Large Language Models", "url": "https://arxiv.org/html/2404.01318v5", "venue": "NeurIPS 2024 Datasets and Benchmarks Track" }, "authors": [ { "name": "Patrick Chao", "affiliation": null }, { "name": "Edoardo Debenedetti", "affiliation": null }, { "name": "Alexander Robey", "affiliation": null }, { "name": "Maksym Andriushchenko", "affiliation": null }, { "name": "Francesco Croce", "affiliation": null }, { "name": "Vikash Sehwag", "affiliation": null }, { "name": "Edgar Dobriban", "affiliation": null }, { "name": "Nicolas Flammarion", "affiliation": null }, { "name": "George J. Pappas", "affiliation": null }, { "name": "Florian Tramèr", "affiliation": null }, { "name": "Hamed Hassani", "affiliation": null }, { "name": "Eric Wong", "affiliation": null } ], "maintainer": "JailbreakBench organization and contributors", "license": "MIT codebase; constituent behavior datasets retain their own attribution requirements", "dataset": { "url": "https://huggingface.co/datasets/JailbreakBench/JBB-Behaviors", "size": 100, "public_test_set": true, "gated": false }, "metric": { "primary": "attack success rate on 100 misuse behaviors (%)", "scoring": "llm-judge", "judge_model": "Llama-3-70B jailbreak judge in the current official package", "aggregation": "percentage of fixed misuse behaviors for which an attack produces a response classified as a successful jailbreak under the declared threat model and query budget", "variants": [ { "name": "benign refusal rate", "description": "Measures whether a defense refuses the 100 benign behavior counterparts; lower over-refusal is better.", "scoring": "llm-judge", "judge_model": "Llama-3-8B refusal judge in the current official package" }, { "name": "attack cost and query budget", "description": "Reports the resources required to obtain successful artifacts; required for fair comparison of adaptive and fixed attacks.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "attack-specific", "cot": "agentic", "temperature": null, "pass_at": null, "tools": false, "harnesses": [ { "name": "JailbreakBench Python package", "url": "https://github.com/JailbreakBench/jailbreakbench", "task_flag": "pip install jailbreakbench" } ] }, "cost_estimate": { "examples": 100, "approx_usd_frontier_model": "Attack-dependent; adaptive methods may make many attacker and target-model calls per behavior." }, "human_baseline": { "score": null, "note": "No human attack-success baseline is defined. A separate 300-example judge-comparison set contains three human labels per response for judge validation." }, "statistical_note": "ASR is conditional on target model version, system prompt, chat template, attack budget, defense, generation settings, and judge. Timeline points use the paper's undefended PAIR attack only so one attack protocol is compared across targets; lower is safer. Timeline dates are target-model public-launch dates, not evaluation dates.", "contamination": { "test_set_public": true, "refresh_cycle": "continuous", "mitigation": "Keep target behaviors fixed for comparability, version jailbreak artifacts and judges, enforce equal target-call budgets, and maintain private behaviors for deployment decisions.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official JailbreakBench leaderboard", "url": "https://jailbreakbench.github.io/", "type": "official" } ], "performance_timeline": [ { "model": "GPT-3.5 Turbo targeted by PAIR", "vendor": "openai", "date": "2023-03-01T00:00:00.000Z", "score": 71, "source": "independent", "source_url": "https://arxiv.org/html/2404.01318v5" }, { "model": "GPT-4 targeted by PAIR", "vendor": "openai", "date": "2023-03-14T00:00:00.000Z", "score": 34, "source": "independent", "source_url": "https://arxiv.org/html/2404.01318v5" }, { "model": "Vicuna-13B targeted by PAIR", "vendor": "lmsys", "date": "2023-03-30T00:00:00.000Z", "score": 69, "source": "independent", "source_url": "https://arxiv.org/html/2404.01318v5" }, { "model": "Llama 2 targeted by PAIR", "vendor": "meta", "date": "2023-07-18T00:00:00.000Z", "score": 0, "source": "independent", "source_url": "https://arxiv.org/html/2404.01318v5" } ], "tags": [ "jailbreaks", "red-teaming", "adversarial-prompts", "defenses", "refusal" ], "last_updated": "2026-09-14T00:00:00.000Z", "changelog": [ { "date": "2026-09-14T00:00:00.000Z", "note": "Added paper-reported PAIR attack-success rates using verified target-model public-launch dates for the timeline x-axis." }, { "date": "2026-09-14T00:00:00.000Z", "note": "Initial page based on arXiv v5, NeurIPS proceedings, the official package, datasets, artifacts, and leaderboard." } ] }, { "name": "LiveCodeBench", "slug": "livecodebench", "capability": "coding", "secondary_capabilities": [], "short_description": "Continuously updated contest problems for time-windowed code generation, execution, test prediction, and self-repair evaluation.", "launch_date": "2024-03-12T00:00:00.000Z", "status": "saturated", "status_evidence": "In the August 2, 2026 LiveCodeBench leaderboard snapshot, Qwen3.7 Max reaches 91.6%, Qwen3.7 Plus 89.6%, and GLM-4.7 84.9%; the leading cluster leaves too little reliable headroom for frontier ranking on this snapshot.", "saturated_date": "2026-08-02T00:00:00.000Z", "successor": null, "lineage": { "parent": null, "variants": [ "livecodebench-code-generation", "livecodebench-code-execution", "livecodebench-test-output-prediction", "livecodebench-self-repair" ] }, "paper": { "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code", "url": "https://arxiv.org/abs/2403.07974", "venue": "ICLR 2025" }, "authors": [ { "name": "Naman Jain", "affiliation": "University of California, Berkeley" }, { "name": "King Han", "affiliation": null }, { "name": "Alex Gu", "affiliation": null }, { "name": "Armando Solar-Lezama", "affiliation": "MIT" }, { "name": "Koushik Sen", "affiliation": "University of California, Berkeley" }, { "name": "Ion Stoica", "affiliation": "University of California, Berkeley" } ], "maintainer": "LiveCodeBench team", "license": "MIT", "dataset": { "url": "https://huggingface.co/livecodebench", "size": 1055, "public_test_set": true, "gated": false }, "metric": { "primary": "pass@1 on code generation (%)", "scoring": "unit-tests", "judge_model": null, "aggregation": "mean over a named release or date window; the official harness also reports pass@5", "variants": [ { "name": "pass@5", "description": "Estimated probability that at least one of five generated solutions passes all tests.", "scoring": "unit-tests", "judge_model": null }, { "name": "code execution / test-output prediction accuracy", "description": "Exact correctness on predicted execution outputs for the corresponding scenarios.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0.2, "pass_at": 1, "tools": false, "harnesses": [ { "name": "LiveCodeBench runner", "url": "https://github.com/LiveCodeBench/LiveCodeBench", "task_flag": "--scenario codegeneration" } ] }, "cost_estimate": { "examples": 1055, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No official aggregate human baseline is reported. Problems come from human programming contests with platform-specific participant statistics, which are not one comparable LiveCodeBench baseline." }, "statistical_note": "Scores are comparable only when leaderboard snapshot, release/date window, scenario, fast-versus-full tests, sampling, timeout, and language policy match. Timeline dates are model launch dates; all six scores belong to the August 2, 2026 leaderboard snapshot.", "contamination": { "test_set_public": true, "refresh_cycle": "continuous", "mitigation": "Problems are collected after publication from LeetCode, AtCoder, and Codeforces; date-window filtering lets evaluators restrict tasks to those newer than a model's training cutoff.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [ { "name": "LiveCodeBench official leaderboard", "url": "https://livecodebench.github.io/leaderboard.html", "type": "official" } ], "performance_timeline": [ { "model": "Qwen3.7 Max (qwen3-7-max; closed)", "vendor": "alibaba", "date": "2026-05-20T00:00:00.000Z", "score": 91.6, "source": "independent", "source_url": "https://livecodebench.github.io/leaderboard" }, { "model": "Qwen3.7 Plus (qwen3-7-plus; closed)", "vendor": "alibaba", "date": "2026-05-26T00:00:00.000Z", "score": 89.6, "source": "independent", "source_url": "https://livecodebench.github.io/leaderboard" }, { "model": "GLM-4.7 (glm-4-7; open weight)", "vendor": "z-ai", "date": "2025-12-22T00:00:00.000Z", "score": 84.9, "source": "independent", "source_url": "https://livecodebench.github.io/leaderboard" }, { "model": "Qwen3.6-27B (open weight)", "vendor": "alibaba", "date": "2026-04-22T00:00:00.000Z", "score": 83.9, "source": "independent", "source_url": "https://livecodebench.github.io/leaderboard" }, { "model": "Qwen3.6-35B-A3B (open weight)", "vendor": "alibaba", "date": "2026-04-15T00:00:00.000Z", "score": 80.4, "source": "independent", "source_url": "https://livecodebench.github.io/leaderboard" }, { "model": "DeepSeek V3 (open weight)", "vendor": "deepseek", "date": "2024-12-26T00:00:00.000Z", "score": 37.6, "source": "independent", "source_url": "https://livecodebench.github.io/leaderboard" } ], "tags": [ "competitive-programming", "rolling", "contamination", "code-generation", "self-repair", "execution" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Reclassified as saturated and replaced the earlier 454-problem snapshot with all six user-supplied August 2, 2026 LiveCodeBench leaderboard scores. Timeline dates use primary-source model launch dates; the separate cross-benchmark Overall and context-window fields are not mixed into the LiveCodeBench score axis." } ] }, { "name": "LongBench", "slug": "longbench", "capability": "long-context", "secondary_capabilities": [ "knowledge", "coding", "reasoning" ], "short_description": "Bilingual multitask suite for long-context understanding across QA, summarization, few-shot learning, synthetic tasks, and code completion.", "launch_date": "2023-08-28T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "LongBench remains a common baseline suite, but its original 5k-15k-heavy, public static task mix has been overtaken for frontier use by LongBench v2/Pro, RULER, MRCR, and 100k+ suites.", "saturated_date": null, "successor": "longbench-pro", "lineage": { "parent": null, "variants": [ "longbench-v2", "longbench-pro" ] }, "paper": { "title": "LongBench: A Bilingual, Multitask Benchmark for Long Context Understanding", "url": "https://arxiv.org/abs/2308.14508", "venue": "ACL 2024" }, "authors": [ { "name": "Yushi Bai", "affiliation": null }, { "name": "Xin Lv", "affiliation": null }, { "name": "Jiajie Zhang", "affiliation": null }, { "name": "Hongchang Lyu", "affiliation": null }, { "name": "Jiankai Tang", "affiliation": null }, { "name": "Zhidian Huang", "affiliation": null }, { "name": "Zhengxiao Du", "affiliation": null }, { "name": "Xiao Liu", "affiliation": null }, { "name": "Aohan Zeng", "affiliation": null }, { "name": "Lei Hou", "affiliation": null }, { "name": "Yuxiao Dong", "affiliation": null }, { "name": "Jie Tang", "affiliation": null }, { "name": "Juanzi Li", "affiliation": null } ], "maintainer": "THUDM", "license": "MIT", "dataset": { "url": "https://huggingface.co/datasets/THUDM/LongBench", "size": 4750, "public_test_set": true, "gated": false }, "metric": { "primary": "average score across task-specific metrics (%)", "scoring": "composite", "judge_model": null, "aggregation": "average over 21 datasets in six categories; task metrics include QA, summarization, classification, retrieval, and code-completion metrics", "variants": [ { "name": "LongBench-E", "description": "Length-balanced subset for analyzing performance across 0-4k, 4k-8k, and 8k+ intervals.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "task-specific", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official LongBench repository", "url": "https://github.com/THUDM/LongBench", "task_flag": null } ] }, "cost_estimate": { "examples": 4750, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No single protocol-comparable human baseline is recorded for the original LongBench aggregate." }, "statistical_note": "The headline aggregate mixes 21 datasets, two natural languages, and code tasks; use category and dataset-level scores for diagnosis. The visual graph on this page deliberately plots only the official LongBench paper Table 3 OverAll column for selected baseline models: the macro-average of Avg over major task categories for few-shot learning, synthetic, and code tasks. It does not plot per-task or per-category scores.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The benchmark is public on Hugging Face and GitHub; contamination risk is substantial for models trained after release.", "risk": "high" }, "languages": [ "english", "chinese", "code" ], "leaderboards": [ { "name": "Official LongBench repository", "url": "https://github.com/THUDM/LongBench", "type": "official" } ], "performance_timeline": [ { "model": "GPT-3.5-Turbo-16k (paper Table 3 OverAll)", "vendor": "openai", "date": "2023-08-28T00:00:00.000Z", "score": 44.7, "source": "independent", "source_url": "https://arxiv.org/abs/2308.14508" }, { "model": "Llama2-7B-chat-4k (paper Table 3 OverAll)", "vendor": "meta", "date": "2023-08-28T00:00:00.000Z", "score": 26.8, "source": "independent", "source_url": "https://arxiv.org/abs/2308.14508" }, { "model": "LongChat-v1.5-7B-32k (paper Table 3 OverAll)", "vendor": "lmsys", "date": "2023-08-28T00:00:00.000Z", "score": 31.6, "source": "independent", "source_url": "https://arxiv.org/abs/2308.14508" }, { "model": "XGen-7B-8k (paper Table 3 OverAll)", "vendor": "salesforce", "date": "2023-08-28T00:00:00.000Z", "score": 25, "source": "independent", "source_url": "https://arxiv.org/abs/2308.14508" }, { "model": "InternLM-7B-8k (paper Table 3 OverAll)", "vendor": "internlm", "date": "2023-08-28T00:00:00.000Z", "score": 22.6, "source": "independent", "source_url": "https://arxiv.org/abs/2308.14508" }, { "model": "ChatGLM2-6B (paper Table 3 OverAll)", "vendor": "zhipu-ai", "date": "2023-08-28T00:00:00.000Z", "score": 25.7, "source": "independent", "source_url": "https://arxiv.org/abs/2308.14508" }, { "model": "ChatGLM2-6B-32k (paper Table 3 OverAll)", "vendor": "zhipu-ai", "date": "2023-08-28T00:00:00.000Z", "score": 41.4, "source": "independent", "source_url": "https://arxiv.org/abs/2308.14508" }, { "model": "Vicuna-v1.5-7B-16k (paper Table 3 OverAll)", "vendor": "lmsys", "date": "2023-08-28T00:00:00.000Z", "score": 30.5, "source": "independent", "source_url": "https://arxiv.org/abs/2308.14508" } ], "tags": [ "long-context", "bilingual", "suite", "summarization", "question-answering", "code" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2026-08-12T00:00:00.000Z", "note": "Added official paper Table 3 OverAll scores to the visual graph and added a visible OverAll-only stats section clarifying that the graph shows only the macro-average over major task categories, not per-task or per-category scores." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Initial long-context page. Timeline omitted to avoid mixing heterogeneous task aggregates and newer LongBench v2/Pro result axes." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Corrected status from active to nearing-saturation because original LongBench is now a baseline/successor-parent, not the strongest current frontier long-context benchmark." } ] }, { "name": "LongBench-Pro", "slug": "longbench-pro", "capability": "long-context", "secondary_capabilities": [ "knowledge", "reasoning", "coding" ], "short_description": "Realistic bilingual long-context benchmark with 1,500 natural samples across 11 primary and 25 secondary tasks from 8k to 256k tokens.", "launch_date": "2026-01-06T00:00:00.000Z", "status": "active", "status_evidence": "Newer than LongBench v1/v2 and designed for more realistic, bilingual, fine-grained long-context analysis across 8k-256k token inputs.", "saturated_date": null, "successor": null, "lineage": { "parent": "longbench", "variants": [] }, "paper": { "title": "LongBench Pro: A More Realistic and Comprehensive Bilingual Long-Context Evaluation Benchmark", "url": "https://arxiv.org/abs/2601.02872", "venue": null }, "authors": [ { "name": "Ziyang Chen", "affiliation": null }, { "name": "Xing Wu", "affiliation": null }, { "name": "Junlong Jia", "affiliation": null }, { "name": "Chaochen Gao", "affiliation": null }, { "name": "Qi Fu", "affiliation": null }, { "name": "Debing Zhang", "affiliation": null }, { "name": "Songlin Hu", "affiliation": null } ], "maintainer": "KCSG Knowledge Computing and Service Group, IIE, CAS", "license": "Apache-2.0", "dataset": { "url": "https://huggingface.co/datasets/caskcsg/LongBench-Pro", "size": 1500, "public_test_set": true, "gated": false }, "metric": { "primary": "normalized task-specific aggregate", "scoring": "composite", "judge_model": null, "aggregation": "task-specific metrics normalized to a 0-100 scale across 1,500 samples, 11 primary tasks, 25 secondary tasks, two languages, six length levels, and four difficulty levels", "variants": [ { "name": "thinking and non-thinking prompts", "description": "The dataset card and repository expose separate prompt fields for direct answers and think-first settings.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "cot", "temperature": null, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official LongBench-Pro code", "url": "https://github.com/caskcsg/longcontext/tree/main/LongBench-Pro", "task_flag": "python main.py" }, { "name": "Hugging Face LongBench-Pro dataset", "url": "https://huggingface.co/datasets/caskcsg/LongBench-Pro", "task_flag": "caskcsg/LongBench-Pro" } ] }, "cost_estimate": { "examples": 1500, "approx_usd_frontier_model": "high for 128k-256k samples, especially with thinking prompts and repeated runs" }, "human_baseline": { "score": null, "note": "The construction pipeline includes expert validation, but no single protocol-comparable human score is recorded here." }, "statistical_note": "The benchmark is intentionally stratified by context requirement, length, and difficulty; aggregate scores should be accompanied by those slices. The visual graph on this page deliberately plots only the official LongBench-Pro paper Overall column for the supplied top-10 ranked models. It does not plot per-task, per-language, per-length, or difficulty-sliced scores; reasoning/thinking status is shown in the table text, not encoded as a separate plotted metric.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The released test file is public on Hugging Face; the benchmark relies on natural documents and expert validation, not on secrecy.", "risk": "high" }, "languages": [ "english", "chinese" ], "leaderboards": [ { "name": "Official LongBench-Pro repository", "url": "https://github.com/caskcsg/longcontext/tree/main/LongBench-Pro", "type": "official" }, { "name": "LongBench-Pro dataset card", "url": "https://huggingface.co/datasets/caskcsg/LongBench-Pro", "type": "official" } ], "performance_timeline": [ { "model": "Gemini-2.5-Pro (paper Overall)", "vendor": "google", "date": "2026-01-06T00:00:00.000Z", "score": 73.42, "source": "independent", "source_url": "https://arxiv.org/abs/2601.02872" }, { "model": "GPT-5 (paper Overall)", "vendor": "openai", "date": "2026-01-06T00:00:00.000Z", "score": 72.61, "source": "independent", "source_url": "https://arxiv.org/abs/2601.02872" }, { "model": "Qwen3-235B-A22B-Thinking-2507 (paper Overall)", "vendor": "alibaba", "date": "2026-01-06T00:00:00.000Z", "score": 66.97, "source": "independent", "source_url": "https://arxiv.org/abs/2601.02872" }, { "model": "Qwen3-Next-80B-A3B-Thinking (paper Overall)", "vendor": "alibaba", "date": "2026-01-06T00:00:00.000Z", "score": 63.95, "source": "independent", "source_url": "https://arxiv.org/abs/2601.02872" }, { "model": "DeepSeek-R1-0528 (paper Overall)", "vendor": "deepseek", "date": "2026-01-06T00:00:00.000Z", "score": 61.89, "source": "independent", "source_url": "https://arxiv.org/abs/2601.02872" }, { "model": "DeepSeek-R1 (paper Overall)", "vendor": "deepseek", "date": "2026-01-06T00:00:00.000Z", "score": 60.07, "source": "independent", "source_url": "https://arxiv.org/abs/2601.02872" }, { "model": "Qwen3-30B-A3B-Thinking-2507 (paper Overall)", "vendor": "alibaba", "date": "2026-01-06T00:00:00.000Z", "score": 59.68, "source": "independent", "source_url": "https://arxiv.org/abs/2601.02872" }, { "model": "Claude-4-Sonnet (paper Overall)", "vendor": "anthropic", "date": "2026-01-06T00:00:00.000Z", "score": 56.07, "source": "independent", "source_url": "https://arxiv.org/abs/2601.02872" }, { "model": "Gemini-2.5-Flash (paper Overall)", "vendor": "google", "date": "2026-01-06T00:00:00.000Z", "score": 55.92, "source": "independent", "source_url": "https://arxiv.org/abs/2601.02872" }, { "model": "MiniMax-M2 (paper Overall)", "vendor": "minimax", "date": "2026-01-06T00:00:00.000Z", "score": 53.21, "source": "independent", "source_url": "https://arxiv.org/abs/2601.02872" } ], "tags": [ "long-context", "bilingual", "realistic-documents", "suite", "thinking" ], "last_updated": "2026-01-06T00:00:00.000Z", "changelog": [ { "date": "2026-01-06T00:00:00.000Z", "note": "Added official paper Overall scores for the supplied top-10 ranked models to the visual graph using the paper publication date, with a visible table marking Thinking versus Mixed status and clarifying that the graph shows only Overall scores." }, { "date": "2026-01-06T00:00:00.000Z", "note": "Initial long-context page. Timeline omitted pending a stable official score axis and model-date policy for the new benchmark." } ] }, { "name": "MATH", "slug": "math", "capability": "maths", "secondary_capabilities": [ "reasoning" ], "short_description": "12,500 AMC/AIME-level competition problems with worked solutions — usually evaluated on its 500-problem MATH-500 subset, the reasoning-model math standard.", "launch_date": "2021-03-05T00:00:00.000Z", "status": "saturated", "status_evidence": "MATH resisted far longer than GSM8K: GPT-4 managed only ~42% on the full set in 2023. But the reasoning-model era ended it — o1 (2024) reached ~94.8% on MATH-500 and later models sit in the 90s, so it no longer discriminates at the frontier. saturated_date 2024-09 marks the o1 inflection. As with GSM8K, saturated ≠ solved: exact-answer scoring hides whether the reasoning was valid, and headroom has moved to AIME and FrontierMath.", "saturated_date": "2024-09-01T00:00:00.000Z", "successor": null, "lineage": { "parent": null, "variants": [ "math-500", "prm800k" ] }, "paper": { "title": "Measuring Mathematical Problem Solving With the MATH Dataset", "url": "https://arxiv.org/abs/2103.03874", "venue": "NeurIPS 2021 Datasets & Benchmarks" }, "authors": [ { "name": "Dan Hendrycks", "affiliation": "UC Berkeley" }, { "name": "Collin Burns", "affiliation": "UC Berkeley" }, { "name": "Jacob Steinhardt", "affiliation": "UC Berkeley" } ], "maintainer": "Dan Hendrycks et al. (UC Berkeley); dataset hendrycks/competition_math. MATH-500 subset & PRM800K: OpenAI (Lightman et al.), repo openai/prm800k", "license": "MIT", "dataset": { "url": "https://huggingface.co/datasets/hendrycks/competition_math", "size": 500, "public_test_set": true, "gated": false }, "metric": { "primary": "accuracy on MATH-500 — math-equivalence match on the boxed answer (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "mean over the 500 MATH-500 problems; the final \\boxed{} answer is graded by mathematical equivalence, not the reasoning", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "cot", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "hendrycks_math / minerva_math" }, { "name": "MATH-500 / PRM800K (official subset)", "url": "https://github.com/openai/prm800k", "task_flag": null } ] }, "cost_estimate": { "examples": 500, "approx_usd_frontier_model": "2–15 on MATH-500 (500 problems; competition math elicits long chains of thought, so reasoning tokens dominate. The full 5,000-problem test set costs ~10x and is why MATH-500 exists)" }, "human_baseline": { "score": 90, "note": "The paper reports a stark human RANGE, not a point: a three-time IMO gold medalist scored 90% while a computer-science PhD student who dislikes math scored ~40%. The gauge uses the IMO-gold 90% as the reference line — the expert ceiling that reasoning models have now surpassed — and the 40% lower bound lives here. For competition math, 'human performance' is nearly meaningless without naming the human; this is the starkest range-not-a-point case on the wiki (cf. AGIEval, GPQA)." }, "statistical_note": "MATH-500 is 500 problems → ±~2.5pt (95% CI) near 90%. The 7 subjects × 5 difficulty levels enable fine-grained analysis, but any single level×subject cell is tiny (tens of problems) — never compare models on 'Level-5 Geometry' alone. As at the saturated frontier, aggregate gaps under a few points are noise; the bigger issue is equivalence-checker disagreement (see Scoring methodology).", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "None, and compounded. Public since 2021, MIT-licensed, with full worked solutions mirrored across the web. Worse: the PRM800K release (2023) put step-level solutions to MATH TRAINING problems into wide circulation as process-supervision data — valuable for research, but more of the benchmark's solution structure in the open. MATH-500's cost motivation is partly entangled with this: it is drawn to avoid the train/PRM overlap and to make grading tractable. Fully static, no refresh. Treat near-ceiling scores as memorization-inclusive.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Papers with Code (MATH)", "url": "https://paperswithcode.com/sota/math-word-problem-solving-on-math", "type": "third-party" } ], "performance_timeline": [ { "model": "GPT-3 175B (full MATH)", "vendor": "openai", "date": "2021-03-05T00:00:00.000Z", "score": 5.6, "source": "independent", "source_url": "https://arxiv.org/abs/2103.03874" }, { "model": "GPT-4 (full MATH)", "vendor": "openai", "date": "2023-03-14T00:00:00.000Z", "score": 42.5, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2303.08774" }, { "model": "o1 (MATH-500)", "vendor": "openai", "date": "2024-09-12T00:00:00.000Z", "score": 94.8, "source": "vendor-reported", "source_url": "https://openai.com/index/learning-to-reason-with-llms/" } ], "tags": [ "maths", "competition", "math-500", "chain-of-thought", "reasoning", "contamination" ], "last_updated": "2026-07-22T00:00:00.000Z", "changelog": [ { "date": "2026-07-22T00:00:00.000Z", "note": "Initial page. Second `maths` benchmark (after GSM8K). Single page covering the MATH dataset with MATH-500 as the headline configuration; dataset.size 500 follows the GPQA size-matches-primary-subset convention (GPQA=198 Diamond) — the same rule, applied consistently. Timeline: GPT-3 175B 5.6% and GPT-4 42.5% on FULL MATH (independent/vendor, arXiv:2103.03874 / 2303.08774), then o1 94.8% on MATH-500 (vendor, OpenAI o1 post). Config (full-MATH vs MATH-500) labelled per point; 2022 Minerva and 2025 frontier models discussed in prose but not plotted (Minerva's headline used maj@k, not pass@1 — avoided to keep the axis pass@1-consistent). Human baseline stored as the IMO-gold 90%; the ~40% math-averse-PhD lower bound is in the note. Cross-links: GSM8K (sibling, easier), MMLU (same lead author, Hendrycks). PRM800K + MATH-500 in lineage.variants (pages not created)." } ] }, { "name": "MathArena", "slug": "matharena", "capability": "maths", "secondary_capabilities": [ "reasoning", "multimodal" ], "short_description": "A continuously refreshed platform evaluating models on newly released math competitions, proofs, research problems, and formalization.", "launch_date": "2025-05-29T00:00:00.000Z", "status": "active", "status_evidence": "MathArena continuously adds newly released competitions and research-derived tasks; even as aggregate expected performance rises, fresh proof, visual, and research tracks preserve measurable headroom.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "MathArena: Evaluating LLMs on Uncontaminated Math Competitions", "url": "https://arxiv.org/abs/2505.23281", "venue": "NeurIPS 2025 Datasets and Benchmarks" }, "authors": [ { "name": "Mislav Balunovic", "affiliation": "ETH Zurich" }, { "name": "Jasper Dekoninck", "affiliation": "ETH Zurich" }, { "name": "Ivo Petrov", "affiliation": "ETH Zurich" }, { "name": "Nikola Jovanovic", "affiliation": "ETH Zurich" }, { "name": "Martin Vechev", "affiliation": "ETH Zurich" } ], "maintainer": "ETH Zurich Secure, Reliable, and Intelligent Systems Lab", "license": "MIT (evaluation repository)", "dataset": { "url": "https://huggingface.co/MathArena", "size": null, "public_test_set": true, "gated": false }, "metric": { "primary": "expected performance across non-deprecated competitions (%)", "scoring": "composite", "judge_model": null, "aggregation": "two-parameter item-response model fitted to per-run correctness, averaged over questions in current non-deprecated competitions; competition leaderboards use their own accuracy or point rules", "variants": [ { "name": "Final-answer competitions", "description": "Four runs per problem are scored using competition-specific exact-answer rules.", "scoring": "exact-match", "judge_model": null }, { "name": "Proof-based competitions", "description": "Solutions receive partial credit under competition rubrics using the platform's judging pipeline.", "scoring": "composite", "judge_model": null }, { "name": "Expected cost", "description": "Average API cost per problem is estimated over the current non-deprecated benchmark mix.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "agentic", "temperature": null, "pass_at": 4, "tools": false, "harnesses": [ { "name": "MathArena official harness", "url": "https://github.com/eth-sri/matharena", "task_flag": "scripts/run.py --comp --models --n 4" } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": "Published per model and competition; varies with the continuously changing problem mix and reasoning effort." }, "human_baseline": { "score": null, "note": "No single human aggregate is meaningful across age-group contests, AIME, IMO proofs, university competitions, Project Euler, recent research questions, and formalization tracks." }, "statistical_note": "Each competition is small and the benchmark membership changes. Use the site's uncertainty intervals and date-stamp every aggregate; per-competition scores are more interpretable than the cross-platform IRT summary.", "contamination": { "test_set_public": true, "refresh_cycle": "continuous", "mitigation": "Models are evaluated immediately after new competition problems are released, before they can plausibly enter subsequent training runs; later public exposure does not preserve cleanliness forever.", "risk": "low" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official MathArena platform", "url": "https://matharena.ai", "type": "official" } ], "performance_timeline": [ { "model": "GPT-5.5 (xhigh, expected performance)", "vendor": "openai", "date": "2026-04-24T00:00:00.000Z", "score": 81.3, "source": "independent", "source_url": "https://matharena.ai/models" }, { "model": "GPT-5.6-Sol (max, expected performance)", "vendor": "openai", "date": "2026-07-09T00:00:00.000Z", "score": 83.1, "source": "independent", "source_url": "https://matharena.ai/models" } ], "tags": [ "maths", "live-benchmark", "competition", "proof", "contamination", "cost" ], "last_updated": "2026-07-29T00:00:00.000Z", "changelog": [ { "date": "2026-07-29T00:00:00.000Z", "note": "Initial page. Timeline uses the platform's current IRT expected-performance axis, not any one competition; values are inherently date- and membership-dependent." } ] }, { "name": "MathVista", "slug": "mathvista", "capability": "multimodal", "secondary_capabilities": [ "maths", "reasoning" ], "short_description": "Visual mathematical reasoning across diagrams, plots, tables, scientific figures, puzzles, and natural images.", "launch_date": "2023-10-03T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "The official leaderboard reported multiple models above the 60.3% crowd-worker baseline by 2024; the public 1,000-item testmini remains useful for diagnosis but is no longer a clean frontier-only signal.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "MathVista: Evaluating Mathematical Reasoning of Foundation Models in Visual Contexts", "url": "https://arxiv.org/abs/2310.02255v3", "venue": "ICLR 2024" }, "authors": [ { "name": "Pan Lu", "affiliation": "UCLA / Microsoft Research" }, { "name": "Hritik Bansal", "affiliation": "UCLA" }, { "name": "Tony Xia et al.", "affiliation": null } ], "maintainer": "MathVista team", "license": "CC BY-SA 4.0 for MathVista's new contributions; source datasets retain their original rights", "dataset": { "url": "https://huggingface.co/datasets/AI4Math/MathVista", "size": 6141, "public_test_set": false, "gated": false }, "metric": { "primary": "accuracy (%)", "scoring": "composite", "judge_model": "GPT-4 answer extractor in the original protocol", "aggregation": "micro accuracy after extracting and normalizing multiple-choice or free-form numeric answers; report testmini (1,000 public-label examples) separately from the 5,141-example hidden-label test set", "variants": [ { "name": "testmini", "description": "Public 1,000-example subset intended for development and lower-cost evaluation.", "scoring": "composite", "judge_model": "GPT-4 answer extractor in the original protocol" }, { "name": "test", "description": "5,141-example standard evaluation split whose labels were withheld at release.", "scoring": "composite", "judge_model": "GPT-4 answer extractor in the original protocol" } ] }, "run_config": { "shots": "zero-shot", "cot": "cot", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official MathVista evaluation", "url": "https://github.com/lupantech/MathVista", "task_flag": "python evaluation/evaluate.py" } ] }, "cost_estimate": { "examples": 1000, "approx_usd_frontier_model": "moderate for testmini; full hidden test is roughly five times larger" }, "human_baseline": { "score": 60.3, "note": "Amazon Mechanical Turk average on testmini; annotators had a high-school diploma or higher. This is not an expert-mathematician ceiling." }, "statistical_note": "Leaderboard comparisons must use the same split, prompt, answer extractor, and tool setting. A 1,000-item testmini accuracy near 50% has a rough 95% binomial margin of about +/-3 points before accounting for heterogeneous tasks.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "The 5,141 test labels were withheld at release, but all testmini labels and source images/questions are public; many items are inherited from 28 existing datasets.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official MathVista leaderboard", "url": "https://mathvista.github.io/", "type": "official" } ], "performance_timeline": [ { "model": "GPT-4V (testmini, original paper)", "vendor": "openai", "date": "2023-10-03T00:00:00.000Z", "score": 49.9, "source": "independent", "source_url": "https://arxiv.org/html/2310.02255v3" }, { "model": "GPT-4o (testmini)", "vendor": "openai", "date": "2024-05-13T00:00:00.000Z", "score": 63.8, "source": "independent", "source_url": "https://github.com/lupantech/MathVista" }, { "model": "OpenAI o1 (testmini)", "vendor": "openai", "date": "2024-09-12T00:00:00.000Z", "score": 73.9, "source": "independent", "source_url": "https://github.com/lupantech/MathVista" } ], "tags": [ "visual-math", "charts", "diagrams", "scientific-figures", "vqa" ], "last_updated": "2026-09-05T00:00:00.000Z", "changelog": [ { "date": "2026-09-05T00:00:00.000Z", "note": "Initial page based on arXiv v3 and the official repository; testmini and hidden-label test protocols are kept distinct." } ] }, { "name": "MBPP", "slug": "mbpp", "capability": "coding", "secondary_capabilities": [], "short_description": "Mostly Basic Programming Problems: 974 crowd-sourced Python synthesis tasks intended for entry-level programmers.", "launch_date": "2021-08-16T00:00:00.000Z", "status": "saturated", "status_evidence": "CodeSOTA's 2026 MBPP pass@1 registry has multiple frontier and open-code models clustered from 89% to 95%, while the short public tasks and three-test protocol remain too weak for current frontier comparison.", "saturated_date": "2023-11-24T00:00:00.000Z", "successor": "mbpp-plus", "lineage": { "parent": null, "variants": [ "mbpp-sanitized", "mbpp-plus" ] }, "paper": { "title": "Program Synthesis with Large Language Models", "url": "https://arxiv.org/abs/2108.07732", "venue": null }, "authors": [ { "name": "Jacob Austin", "affiliation": "Google Research" }, { "name": "Augustus Odena", "affiliation": "Google Research" }, { "name": "Maxwell Nye", "affiliation": "Massachusetts Institute of Technology" }, { "name": "Charles Sutton", "affiliation": "Google Research" } ], "maintainer": "Google Research", "license": "CC-BY-4.0", "dataset": { "url": "https://huggingface.co/datasets/google-research-datasets/mbpp", "size": 500, "public_test_set": true, "gated": false }, "metric": { "primary": "pass@1 on the 500-task test split (%)", "scoring": "unit-tests", "judge_model": null, "aggregation": "fraction of test tasks for which a generated Python program passes the supplied tests", "variants": [ { "name": "any-sample solve rate", "description": "Original-paper scaling analyses also count a task solved if any of many sampled programs passes; this is not pass@1.", "scoring": "unit-tests", "judge_model": null } ] }, "run_config": { "shots": "3-shot (original paper; modern harnesses vary)", "cot": "direct", "temperature": null, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Google Research MBPP data", "url": "https://github.com/google-research/google-research/tree/master/mbpp", "task_flag": null }, { "name": "EvalPlus", "url": "https://github.com/evalplus/evalplus", "task_flag": "mbpp" } ] }, "cost_estimate": { "examples": 500, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No controlled aggregate human baseline was published. Tasks were designed to be solvable by entry-level programmers, and a subset was edited and hand-verified, but neither is a measured score." }, "statistical_note": "At 90% over 500 tasks, the rough 95% binomial interval is about ±2.6 points; protocol and test-quality variation can be larger. Timeline dates are model launch dates; source_url values point to the CodeSOTA MBPP pass@1 registry snapshot.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The paper found little overlap with its own pretraining corpus, but all prompts, reference code, and assertions are now public and static.", "risk": "high" }, "languages": [ "python" ], "leaderboards": [ { "name": "CodeSOTA MBPP leaderboard", "url": "https://www.codesota.com/benchmark/mbpp", "type": "third-party" }, { "name": "EvalPlus leaderboard", "url": "https://evalplus.github.io/leaderboard.html", "type": "third-party" } ], "performance_timeline": [ { "model": "o4-mini (CodeSOTA MBPP pass@1)", "vendor": "openai", "date": "2025-04-16T00:00:00.000Z", "score": 94.9, "source": "independent", "source_url": "https://www.codesota.com/benchmark/mbpp" }, { "model": "o3-mini (CodeSOTA MBPP pass@1)", "vendor": "openai", "date": "2025-01-31T00:00:00.000Z", "score": 93.3, "source": "independent", "source_url": "https://www.codesota.com/benchmark/mbpp" }, { "model": "Claude Opus 4 (CodeSOTA MBPP pass@1)", "vendor": "anthropic", "date": "2025-05-22T00:00:00.000Z", "score": 92, "source": "independent", "source_url": "https://www.codesota.com/benchmark/mbpp" }, { "model": "Claude 3.5 Sonnet (Oct 2024; CodeSOTA MBPP pass@1)", "vendor": "anthropic", "date": "2024-10-22T00:00:00.000Z", "score": 91, "source": "independent", "source_url": "https://www.codesota.com/benchmark/mbpp" }, { "model": "GPT-4.1 (CodeSOTA MBPP pass@1)", "vendor": "openai", "date": "2025-04-14T00:00:00.000Z", "score": 90.9, "source": "independent", "source_url": "https://www.codesota.com/benchmark/mbpp" }, { "model": "Qwen2.5-Coder-32B-Instruct (CodeSOTA MBPP pass@1)", "vendor": "alibaba", "date": "2024-11-12T00:00:00.000Z", "score": 90.2, "source": "independent", "source_url": "https://www.codesota.com/benchmark/mbpp" }, { "model": "Qwen2.5-Coder 32B (CodeSOTA MBPP pass@1)", "vendor": "alibaba", "date": "2024-11-12T00:00:00.000Z", "score": 90.2, "source": "independent", "source_url": "https://www.codesota.com/benchmark/mbpp" }, { "model": "Claude Sonnet 4 (CodeSOTA MBPP pass@1)", "vendor": "anthropic", "date": "2025-05-22T00:00:00.000Z", "score": 89.6, "source": "independent", "source_url": "https://www.codesota.com/benchmark/mbpp" }, { "model": "DeepSeek-Coder-V2-Instruct (CodeSOTA MBPP pass@1)", "vendor": "deepseek", "date": "2024-06-14T00:00:00.000Z", "score": 89.4, "source": "independent", "source_url": "https://www.codesota.com/benchmark/mbpp" }, { "model": "DeepSeek-V3 (CodeSOTA MBPP pass@1)", "vendor": "deepseek", "date": "2024-12-26T00:00:00.000Z", "score": 89.3, "source": "independent", "source_url": "https://www.codesota.com/benchmark/mbpp" } ], "tags": [ "python", "entry-level", "function-synthesis", "unit-tests", "crowdsourced" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Updated MBPP scores from CodeSOTA's pass@1 leaderboard top ten and changed timeline dates to model launch dates: DeepSeek-Coder-V2-Instruct 2024-06-14, Claude 3.5 Sonnet Oct 2024 2024-10-22, Qwen2.5-Coder 2024-11-12, DeepSeek-V3 2024-12-26, o3-mini 2025-01-31, GPT-4.1 2025-04-14, o4-mini 2025-04-16, and Claude Opus/Sonnet 4 2025-05-22." }, { "date": "2026-08-03T00:00:00.000Z", "note": "Initial page. dataset.size follows the standard 500-task test split. Timeline uses only 3-shot pass@1 on that split (Qwen2.5-Coder-32B Base and DeepSeek-V3); the original paper's 59.6% any-of-80-samples result remains prose-only because it is not pass@1." } ] }, { "name": "MBPP+", "slug": "mbpp-plus", "capability": "coding", "secondary_capabilities": [], "short_description": "A cleaned 378-task MBPP evaluation with roughly 35 times more tests for stricter Python functional correctness.", "launch_date": "2023-11-24T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "The 2026-07-13 BenchmarkList snapshot tops out at 80.2% pass@1, while the small public task set, provider-sourced reporting, and dataset-version sensitivity limit clean frontier discrimination.", "saturated_date": null, "successor": null, "lineage": { "parent": "mbpp", "variants": [] }, "paper": { "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation", "url": "https://arxiv.org/abs/2305.01210", "venue": "NeurIPS 2023 (EvalPlus framework; MBPP+ added later)" }, "authors": [ { "name": "Jiawei Liu", "affiliation": "University of Illinois Urbana-Champaign" }, { "name": "Chunqiu Steven Xia", "affiliation": "University of Illinois Urbana-Champaign" }, { "name": "Yuyao Wang", "affiliation": "Nanjing University" }, { "name": "Lingming Zhang", "affiliation": "University of Illinois Urbana-Champaign" } ], "maintainer": "EvalPlus team", "license": "Apache-2.0 (EvalPlus repository)", "dataset": { "url": "https://huggingface.co/datasets/evalplus/mbppplus", "size": 378, "public_test_set": true, "gated": false }, "metric": { "primary": "MBPP+ pass@1 (%)", "scoring": "unit-tests", "judge_model": null, "aggregation": "fraction of the current 378 tasks for which one generated completion passes all base and plus tests", "variants": [ { "name": "MBPP base pass@1", "description": "The same completions evaluated against original MBPP assertions for a paired test-strength comparison.", "scoring": "unit-tests", "judge_model": null } ] }, "run_config": { "shots": "3-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "EvalPlus", "url": "https://github.com/evalplus/evalplus", "task_flag": "mbpp" } ] }, "cost_estimate": { "examples": 378, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No controlled human pass@1 baseline is published for the cleaned MBPP+ set; reference implementations serve as test oracles, not a timed human evaluation." }, "statistical_note": "At 90% over 378 tasks, the rough 95% binomial interval is about ±3.0 points. The pre-v0.3.0 removal of 21 broken tasks changed expected scores by about four points. Timeline dates are official model launch dates; scores come from BenchmarkList's 2026-07-13 snapshot and are not claimed as launch-day evaluations.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Expanded tests catch more overfitting, but both the inherited MBPP prompts and MBPP+ tests are public and versioned.", "risk": "high" }, "languages": [ "python" ], "leaderboards": [ { "name": "Official EvalPlus leaderboard", "url": "https://evalplus.github.io/leaderboard.html", "type": "official" }, { "name": "BenchmarkList MBPP+ leaderboard snapshot", "url": "https://benchmarklist.com/benchmarks/mbpp/", "type": "third-party" } ], "performance_timeline": [ { "model": "O1 Preview (Sept 2024)", "vendor": "openai", "date": "2024-09-12T00:00:00.000Z", "score": 80.2, "source": "vendor-reported", "source_url": "https://benchmarklist.com/benchmarks/mbpp/" }, { "model": "O1 Mini (Sept 2024)", "vendor": "openai", "date": "2024-09-12T00:00:00.000Z", "score": 78.8, "source": "vendor-reported", "source_url": "https://benchmarklist.com/benchmarks/mbpp/" }, { "model": "Qwen2.5-Coder-32B-Instruct", "vendor": "alibaba", "date": "2024-11-12T00:00:00.000Z", "score": 77, "source": "vendor-reported", "source_url": "https://benchmarklist.com/benchmarks/mbpp/" }, { "model": "DeepSeek-Coder-V2-Instruct", "vendor": "deepseek", "date": "2024-06-17T00:00:00.000Z", "score": 75.1, "source": "vendor-reported", "source_url": "https://benchmarklist.com/benchmarks/mbpp/" }, { "model": "Gemini 1.5 Pro 002", "vendor": "google", "date": "2024-09-24T00:00:00.000Z", "score": 74.6, "source": "vendor-reported", "source_url": "https://benchmarklist.com/benchmarks/mbpp/" } ], "tags": [ "python", "entry-level", "function-synthesis", "unit-tests", "fuzzing" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Reduced the BenchmarkList update to the user-requested top five MBPP+ models only and dated each score point by the model's official launch date; score provenance remains the 2026-07-13 BenchmarkList snapshot." }, { "date": "2026-08-03T00:00:00.000Z", "note": "Initial page. Uses the current 378-task MBPP+ set. Timeline is restricted to the single EvalPlus snapshot in Qwen2.5-Coder Table 16: greedy MBPP+ pass@1 for Qwen2.5-Coder-7B-Instruct (71.7), independently re-evaluated DeepSeek-Coder-V2-Instruct (75.1), and Qwen2.5-Coder-32B-Instruct (75.1)." } ] }, { "name": "Mercury", "slug": "mercury", "capability": "coding", "secondary_capabilities": [], "short_description": "Python code-synthesis benchmark that rewards both functional correctness and runtime efficiency against real solution distributions.", "launch_date": "2024-02-12T00:00:00.000Z", "status": "deprecated", "status_evidence": "Mercury is a static 256-task public LeetCode-derived evaluation with no official maintained leaderboard or refreshed test set. Beyond remains useful for efficiency research, but the 2024 launch snapshot is no longer a frontier comparison.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "mercury-train", "mercury-eval" ] }, "paper": { "title": "Mercury: A Code Efficiency Benchmark for Code Large Language Models", "url": "https://arxiv.org/abs/2402.07844", "venue": "NeurIPS 2024 Datasets and Benchmarks Track" }, "authors": [ { "name": "Mingzhe Du", "affiliation": "Nanyang Technological University / National University of Singapore" }, { "name": "Anh Tuan Luu", "affiliation": "Nanyang Technological University" }, { "name": "Bin Ji", "affiliation": "National University of Singapore" }, { "name": "Qian Liu", "affiliation": null }, { "name": "See-Kiong Ng", "affiliation": "National University of Singapore" } ], "maintainer": "Mercury authors (Elfsong/Mercury)", "license": null, "dataset": { "url": "https://huggingface.co/datasets/Elfsong/Mercury", "size": 256, "public_test_set": true, "gated": false }, "metric": { "primary": "Beyond with 5 sampled solutions per task (%)", "scoring": "composite", "judge_model": null, "aggregation": "mean runtime-percentile-weighted pass score across the 256-task Mercury-eval split; functionally incorrect programs receive no efficiency credit", "variants": [ { "name": "Pass@1", "description": "Fraction of generated programs that pass the generated correctness tests, without runtime weighting.", "scoring": "unit-tests", "judge_model": null } ] }, "run_config": { "shots": "zero-shot code synthesis", "cot": "direct", "temperature": null, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Mercury official harness", "url": "https://github.com/Elfsong/Mercury", "task_flag": null } ] }, "cost_estimate": { "examples": 256, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No official human Beyond@1 baseline is reported. The comparison reference is a distribution of accepted human-written LeetCode solutions, not people answering the benchmark under controlled conditions." }, "statistical_note": "Beyond depends on runtime percentiles and the execution environment. Compare models only under the same hardware, sandbox, test generators, timeout policy, sample count, and Mercury revision.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The authors deduplicate collected solutions and use generated tests, but tasks derive from public LeetCode problems and the released evaluation data are static.", "risk": "high" }, "languages": [ "python" ], "leaderboards": [], "performance_timeline": [ { "model": "CodeLlama-34B base (Mercury-eval Overall Beyond, 5 samples)", "vendor": "meta", "date": "2023-08-24T00:00:00.000Z", "score": 42.4, "source": "independent", "source_url": "https://arxiv.org/html/2402.07844#S4.T3" }, { "model": "DeepSeek-Coder-33B base (Mercury-eval Overall Beyond, 5 samples)", "vendor": "deepseek", "date": "2023-11-02T00:00:00.000Z", "score": 48.53, "source": "independent", "source_url": "https://arxiv.org/html/2402.07844#S4.T3" }, { "model": "StarCoder2-15B base (Mercury-eval Overall Beyond, 5 samples)", "vendor": "bigcode", "date": "2024-02-28T00:00:00.000Z", "score": 49.17, "source": "independent", "source_url": "https://arxiv.org/html/2402.07844#S4.T3" }, { "model": "CodeQwen1.5-7B base (Mercury-eval Overall Beyond, 5 samples)", "vendor": "qwen", "date": "2024-04-18T00:00:00.000Z", "score": 47.78, "source": "independent", "source_url": "https://arxiv.org/html/2402.07844#S4.T3" } ], "tags": [ "code-generation", "efficiency", "runtime", "leetcode", "python", "beyond-at-k" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Reclassified as deprecated. Added a fixed launch-era series from paper Table 3: untuned base models, 256-task Mercury-eval, Overall Beyond, five samples per task. DPO/SFT variants and Pass scores remain off-axis." } ] }, { "name": "MGSM", "slug": "mgsm", "capability": "maths", "secondary_capabilities": [ "reasoning" ], "short_description": "A human-translated 250-problem GSM8K subset for comparing grade-school mathematical reasoning across ten languages.", "launch_date": "2022-10-06T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "The small, static translations are broadly supported by modern multilingual models, while large language gaps and prompt sensitivity still make MGSM useful diagnostically; newer perturbation-based successors test robustness more directly.", "saturated_date": null, "successor": null, "lineage": { "parent": "gsm8k", "variants": [] }, "paper": { "title": "Language Models are Multilingual Chain-of-Thought Reasoners", "url": "https://arxiv.org/abs/2210.03057", "venue": "ICLR 2023" }, "authors": [ { "name": "Freda Shi", "affiliation": null }, { "name": "Mirac Suzgun", "affiliation": null }, { "name": "Markus Freitag", "affiliation": "Google Research" }, { "name": "Jason Wei", "affiliation": "Google Research" } ], "maintainer": "Google Research", "license": "Apache-2.0 (repository)", "dataset": { "url": "https://github.com/google-research/url-nlp/tree/main/mgsm", "size": 250, "public_test_set": true, "gated": false }, "metric": { "primary": "macro-average exact-answer accuracy across languages (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "compute numeric exact-match accuracy separately for each language, then macro-average the ten translated-language scores", "variants": [ { "name": "Native-language chain of thought", "description": "Question, exemplars, reasoning cue, and response use the target language.", "scoring": "exact-match", "judge_model": null }, { "name": "English chain of thought", "description": "The target-language question is solved with English exemplars or English reasoning.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "few-shot exemplars are the original protocol; maintained harness also supports zero-shot direct and CoT variants", "cot": "cot", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "lm-evaluation-harness (native-language CoT)", "url": "https://github.com/EleutherAI/lm-evaluation-harness/tree/main/lm_eval/tasks/mgsm", "task_flag": "mgsm_cot_native" }, { "name": "lm-evaluation-harness (direct)", "url": "https://github.com/EleutherAI/lm-evaluation-harness/tree/main/lm_eval/tasks/mgsm", "task_flag": "mgsm_direct" } ] }, "cost_estimate": { "examples": 2500, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No single aggregate human baseline was published; the arithmetic is grade-school level, while translation and language competence are part of the task." }, "statistical_note": "Each language has only 250 paired items: near 50% accuracy the per-language 95% binomial interval is about +/-6.2 points. Cross-language errors are correlated because every language translates the same underlying problems.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Human translation reduces exact English-string overlap but the underlying GSM8K questions, all translations, targets, and exemplars are public.", "risk": "high" }, "languages": [ "bengali", "chinese", "french", "german", "japanese", "russian", "spanish", "swahili", "telugu", "thai" ], "leaderboards": [], "performance_timeline": [ { "model": "PaLM 540B (translate-to-English CoT)", "vendor": "google", "date": "2023-12-06T00:00:00.000Z", "score": 74.7, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2312.11805" } ], "tags": [ "maths", "multilingual", "gsm8k", "chain-of-thought", "translation" ], "last_updated": "2026-07-29T00:00:00.000Z", "changelog": [ { "date": "2026-07-29T00:00:00.000Z", "note": "Initial page. Records 250 semantic items translated into ten target languages; the English file and cost count are documented separately rather than inflating dataset.size." } ] }, { "name": "MM-SafetyBench", "slug": "mm-safetybench", "capability": "safety-alignment", "secondary_capabilities": [ "multimodal" ], "short_description": "A 5,040-pair benchmark testing whether query-relevant images can bypass multimodal model safeguards.", "launch_date": "2023-11-29T00:00:00.000Z", "status": "deprecated", "status_evidence": "MM-SafetyBench established the image-based jailbreak threat, but its static public scenarios and early-model protocol have been overtaken by broader multimodal safety suites and later variants.", "saturated_date": null, "successor": "mm-safetybench-plus-plus", "lineage": { "parent": null, "variants": [ "mm-safetybench-tiny", "mm-safetybench-plus-plus" ] }, "paper": { "title": "MM-SafetyBench: A Benchmark for Safety Evaluation of Multimodal Large Language Models", "url": "https://arxiv.org/html/2311.17600v5", "venue": "ECCV 2024" }, "authors": [ { "name": "Xin Liu", "affiliation": null }, { "name": "Yichen Zhu", "affiliation": null }, { "name": "Jindong Gu", "affiliation": null }, { "name": "Yunshi Lan", "affiliation": null }, { "name": "Chao Yang", "affiliation": null }, { "name": "Yu Qiao", "affiliation": null } ], "maintainer": "MM-SafetyBench authors", "license": "CC-BY-NC-4.0 research-only dataset; generated assets also follow upstream model licenses", "dataset": { "url": "https://github.com/isXinLiu/MM-SafetyBench", "size": 5040, "public_test_set": true, "gated": false }, "metric": { "primary": "attack success rate (%) - lower is safer", "scoring": "llm-judge", "judge_model": "GPT-4 in the paper protocol", "aggregation": "unsafe responses divided by safe plus unsafe responses, reported by risk scenario and image construction; text-only and image-based settings remain separate", "variants": [ { "name": "SD", "description": "Query-relevant image generated from extracted unsafe key phrases.", "scoring": "llm-judge", "judge_model": "GPT-4 in the paper protocol" }, { "name": "SD+TYPO", "description": "Query-relevant image augmented with typography carrying the malicious query.", "scoring": "llm-judge", "judge_model": "GPT-4 in the paper protocol" }, { "name": "TYPO", "description": "Typography-only image condition used to separate visual text effects from query-relevant imagery.", "scoring": "llm-judge", "judge_model": "GPT-4 in the paper protocol" } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official MM-SafetyBench code", "url": "https://github.com/isXinLiu/MM-SafetyBench", "task_flag": "evaluation.py" } ] }, "cost_estimate": { "examples": 5040, "approx_usd_frontier_model": "High: multimodal target inference plus judge-model calls for the full set." }, "human_baseline": { "score": null, "note": "No protocol-comparable human attack-success baseline is reported; humans are not safety-aligned multimodal target models." }, "statistical_note": "ASR depends on the scenario, image variant, target model, model safety policy, prompt formatting, and GPT-4 judge implementation. Timeline points use the comparable tiny-set SD+TYPO condition; lower is safer. Timeline dates are target-model public-launch dates, not evaluation dates.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Version question and image files, keep image conditions separate, use response-only judging where possible, and add private image attacks generated after the target model's training cutoff.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [], "performance_timeline": [ { "model": "LLaVA-1.5 / Vicuna-7B (tiny SD+TYPO ASR)", "vendor": "llava", "date": "2023-10-05T00:00:00.000Z", "score": 77.33, "source": "independent", "source_url": "https://arxiv.org/html/2311.17600v5" }, { "model": "LLaVA-Phi / Phi-2-2.7B (tiny SD+TYPO ASR)", "vendor": "llava", "date": "2024-01-03T00:00:00.000Z", "score": 66.72, "source": "independent", "source_url": "https://arxiv.org/html/2311.17600v5" } ], "tags": [ "multimodal-safety", "visual-jailbreak", "images", "attack-success-rate", "toxicity" ], "last_updated": "2026-09-14T00:00:00.000Z", "changelog": [ { "date": "2026-09-14T00:00:00.000Z", "note": "Added comparable tiny-set SD+TYPO attack-success rates using verified target-model public-launch dates for the timeline x-axis." }, { "date": "2026-09-14T00:00:00.000Z", "note": "Initial page based on arXiv v5, the ECCV release, and official code, questions, image assets, and evaluator." } ] }, { "name": "MME", "slug": "mme", "capability": "multimodal", "secondary_capabilities": [ "reasoning", "knowledge" ], "short_description": "Image benchmark with paired yes/no questions across 10 perception and 4 cognition subtasks.", "launch_date": "2023-06-23T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "Still widely supported as a compact diagnostic, but its small, public, static yes/no tasks provide limited frontier headroom; the same team now maintains harder MME-RealWorld and MME-CoT suites.", "saturated_date": null, "successor": "mme-realworld", "lineage": { "parent": null, "variants": [ "mme-realworld", "mme-cot" ] }, "paper": { "title": "MME: A Comprehensive Evaluation Benchmark for Multimodal Large Language Models", "url": "https://arxiv.org/abs/2306.13394v5", "venue": "NeurIPS 2025 Datasets and Benchmarks Track" }, "authors": [ { "name": "Chaoyou Fu", "affiliation": "Nanjing University" }, { "name": "Peixian Chen", "affiliation": "Tencent Youtu Lab" }, { "name": "Yunhang Shen et al.", "affiliation": null } ], "maintainer": "MME Benchmarks / Chaoyou Fu and collaborators", "license": null, "dataset": { "url": "https://github.com/BradyFU/Awesome-Multimodal-Large-Language-Models/tree/Evaluation", "size": 2374, "public_test_set": true, "gated": false }, "metric": { "primary": "perception score (0-2000) and cognition score (0-800)", "scoring": "composite", "judge_model": null, "aggregation": "For each subtask, accuracy plus accuracy+; accuracy+ requires both the positive and negative question for an image to be correct. Sum 10 perception subtasks and 4 cognition subtasks separately.", "variants": [ { "name": "Subtask score", "description": "Accuracy plus accuracy+, ranging from 0 to 200 for each subtask.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official MME evaluation code", "url": "https://github.com/BradyFU/Awesome-Multimodal-Large-Language-Models/tree/Evaluation", "task_flag": "MME" } ] }, "cost_estimate": { "examples": 2374, "approx_usd_frontier_model": "low to moderate; 2,374 short image-question prompts" }, "human_baseline": { "score": null, "note": "No protocol-comparable aggregate human baseline is reported; the paired questions are manually authored and intended to be straightforward for people." }, "statistical_note": "Do not combine perception and cognition into an undocumented percentage. The two official totals have different maxima, and accuracy+ makes paired failures count differently from ordinary question accuracy.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The authors manually wrote all question-answer pairs and reused only images, not original annotations, from public datasets where applicable. Public release after launch still permits later training exposure.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official MME leaderboard", "url": "https://github.com/BradyFU/Awesome-Multimodal-Large-Language-Models/tree/Evaluation", "type": "official" } ], "performance_timeline": [], "tags": [ "images", "perception", "cognition", "yes-no", "hallucination", "ocr" ], "last_updated": "2026-09-05T00:00:00.000Z", "changelog": [ { "date": "2026-09-05T00:00:00.000Z", "note": "Initial page based on arXiv v5; classified as nearing saturation while preserving separate perception and cognition score axes." } ] }, { "name": "MMLU", "slug": "mmlu", "capability": "knowledge", "secondary_capabilities": [], "short_description": "57-subject multiple-choice exam spanning STEM, humanities, and social sciences — the defining knowledge benchmark of the 2020–2024 era.", "launch_date": "2020-09-07T00:00:00.000Z", "status": "saturated", "status_evidence": "Frontier models cluster at 86–92% since 2024; ~2–6% label errors put the effective ceiling near 90%, so top-model gaps are within noise.", "saturated_date": "2024-06-01T00:00:00.000Z", "successor": "mmlu-pro", "lineage": { "parent": null, "variants": [ "mmlu-pro", "mmlu-redux", "global-mmlu", "mmmlu" ] }, "paper": { "title": "Measuring Massive Multitask Language Understanding", "url": "https://arxiv.org/abs/2009.03300", "venue": "ICLR 2021" }, "authors": [ { "name": "Dan Hendrycks", "affiliation": "UC Berkeley" }, { "name": "Collin Burns", "affiliation": "Columbia University" }, { "name": "Steven Basart", "affiliation": "UChicago" }, { "name": "Andy Zou", "affiliation": "UC Berkeley" }, { "name": "Mantas Mazeika", "affiliation": "UIUC" }, { "name": "Dawn Song", "affiliation": "UC Berkeley" }, { "name": "Jacob Steinhardt", "affiliation": "UC Berkeley" } ], "maintainer": "Center for AI Safety (CAIS) / Dan Hendrycks", "license": "MIT", "dataset": { "url": "https://huggingface.co/datasets/cais/mmlu", "size": 14042, "public_test_set": true, "gated": false }, "metric": { "primary": "accuracy (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "macro average across 57 subjects (some reports use micro average over all questions — the two differ by up to ~1pt; always check which one a leaderboard uses)", "variants": [] }, "run_config": { "shots": "5-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "mmlu" }, { "name": "HELM", "url": "https://crfm.stanford.edu/helm", "task_flag": "mmlu" }, { "name": "Inspect", "url": "https://inspect.aisi.org.uk", "task_flag": "mmlu" } ] }, "cost_estimate": { "examples": 14042, "approx_usd_frontier_model": "10–40 (short prompts, single-token answers; 5-shot prompts dominate token count)" }, "human_baseline": { "score": 89.8, "note": "Estimated expert-level accuracy from the original paper (per-subject specialists, e.g. 87% for a professional-level test-taker on morality, ~90% averaged). Unspecialized human crowdworkers scored only 34.5%." }, "statistical_note": "14,042 questions → tight CIs (±~0.5pt at 85%), but per-subject splits are small (some subjects ~100 questions → ±~8pt); never compare models on a single subject. Label errors (~2–6%) dwarf sampling error at the frontier.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "None. Test set has been fully public on GitHub/HuggingFace since 2020 and appears verbatim in web-scale crawls; most modern pretraining corpora contain it despite decontamination attempts.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Papers with Code", "url": "https://paperswithcode.com/sota/multi-task-language-understanding-on-mmlu", "type": "third-party" }, { "name": "HELM Classic", "url": "https://crfm.stanford.edu/helm/classic/latest", "type": "third-party" } ], "performance_timeline": [ { "model": "GPT-3 175B", "vendor": "openai", "date": "2020-09-07T00:00:00.000Z", "score": 43.9, "source": "independent", "source_url": "https://arxiv.org/abs/2009.03300" }, { "model": "Gopher 280B", "vendor": "deepmind", "date": "2021-12-08T00:00:00.000Z", "score": 60, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2112.11446" }, { "model": "Chinchilla 70B", "vendor": "deepmind", "date": "2022-03-29T00:00:00.000Z", "score": 67.6, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2203.15556" }, { "model": "PaLM 540B", "vendor": "google", "date": "2022-04-05T00:00:00.000Z", "score": 69.3, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2204.02311" }, { "model": "GPT-4", "vendor": "openai", "date": "2023-03-14T00:00:00.000Z", "score": 86.4, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2303.08774" }, { "model": "Claude 3 Opus", "vendor": "anthropic", "date": "2024-03-04T00:00:00.000Z", "score": 86.8, "source": "vendor-reported", "source_url": "https://www.anthropic.com/news/claude-3-family" }, { "model": "GPT-4o", "vendor": "openai", "date": "2024-05-13T00:00:00.000Z", "score": 88.7, "source": "vendor-reported", "source_url": "https://openai.com/index/hello-gpt-4o/" }, { "model": "Llama 3.1 405B", "vendor": "meta", "date": "2024-07-23T00:00:00.000Z", "score": 88.6, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2407.21783" } ], "tags": [ "multiple-choice", "world-knowledge", "academic-exams", "foundational" ], "last_updated": "2026-07-21T00:00:00.000Z", "changelog": [ { "date": "2026-07-21T00:00:00.000Z", "note": "Initial page. Timeline ends at 2024 frontier cluster — post-2024 models rarely report MMLU as a headline number, which is itself the saturation signal." } ] }, { "name": "MMLU-Pro", "slug": "mmlu-pro", "capability": "knowledge", "secondary_capabilities": [ "reasoning" ], "short_description": "MMLU rebuilt to fix its flaws: 10 options instead of 4, trivia and label noise filtered out, and reasoning-heavy questions that reward chain-of-thought.", "launch_date": "2024-06-03T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "Frontier models cluster in the high 80s on MMLU-Pro by 2025–26 (leaderboards put the top ~89–90%), but cleaner labels and 10-option questions push the effective ceiling above MMLU's ~90%, so it still separates strong models where MMLU cannot. The 16–33 point accuracy drop it induced over MMLU is eroding; the trajectory points at saturation within a generation or two, hence nearing-saturation rather than active.", "saturated_date": null, "successor": null, "lineage": { "parent": "mmlu", "variants": [] }, "paper": { "title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark", "url": "https://arxiv.org/abs/2406.01574", "venue": "NeurIPS 2024 Datasets & Benchmarks (Spotlight)" }, "authors": [ { "name": "Yubo Wang", "affiliation": "University of Waterloo / TIGER-Lab" }, { "name": "Xueguang Ma", "affiliation": "University of Waterloo / TIGER-Lab" }, { "name": "Wenhu Chen", "affiliation": "University of Waterloo / TIGER-Lab" } ], "maintainer": "TIGER-Lab (University of Waterloo); repo TIGER-AI-Lab/MMLU-Pro", "license": "CC-BY-4.0", "dataset": { "url": "https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro", "size": 12032, "public_test_set": true, "gated": false }, "metric": { "primary": "accuracy, 10-option multiple choice (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "mean accuracy over 12,032 questions; also reported per-category across the 14 disciplines", "variants": [] }, "run_config": { "shots": "5-shot", "cot": "cot", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "MMLU-Pro (official)", "url": "https://github.com/TIGER-AI-Lab/MMLU-Pro", "task_flag": null }, { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "mmlu_pro" } ] }, "cost_estimate": { "examples": 12032, "approx_usd_frontier_model": "10–40 (12,032 questions, and the standard 5-shot chain-of-thought protocol means long reasoning traces per item — several times MMLU's per-question token cost)" }, "human_baseline": { "score": null, "note": "No measured human baseline. MMLU-Pro's difficulty was calibrated by model-side filtering (removing questions strong models found trivial and adding reasoning-heavy ones), not by testing humans — so unlike its parent MMLU (famous for its 89.8% expert baseline framing) and unlike GPQA (built around expert vs non-expert gaps), the successor here has no human reference line. A notable irony worth flagging when reading 'above/below human' claims: there is no human number to compare against." }, "statistical_note": "12,032 questions give tight aggregate CIs (±~0.6pt at 85%), and the 10-option format lowers the random-guess floor from 25% to 10%, so scores reflect knowledge more than test-taking. But per-category splits (14 disciplines) are much smaller — do not compare models on a single discipline. As frontier scores compress into the high 80s, top-model gaps under a couple of points are again approaching noise.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Public and ungated on HuggingFace, and fully static with no refreshed variant. Contamination risk is lower than MMLU's mainly because it launched in mid-2024 — less crawl exposure so far — but it is rising, and many questions are drawn from public STEM problem sets (e.g. exam and textbook sources) that may themselves appear in training data. Treat it as medium now, trending up.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [ { "name": "MMLU-Pro (official, HuggingFace)", "url": "https://huggingface.co/spaces/TIGER-Lab/MMLU-Pro", "type": "official" }, { "name": "Artificial Analysis (MMLU-Pro)", "url": "https://artificialanalysis.ai/evaluations/mmlu-pro", "type": "third-party" } ], "performance_timeline": [ { "model": "GPT-4-Turbo", "vendor": "openai", "date": "2024-04-09T00:00:00.000Z", "score": 63.7, "source": "independent", "source_url": "https://arxiv.org/abs/2406.01574" }, { "model": "Claude 3 Opus", "vendor": "anthropic", "date": "2024-03-04T00:00:00.000Z", "score": 68.5, "source": "independent", "source_url": "https://arxiv.org/abs/2406.01574" }, { "model": "Gemini 1.5 Pro", "vendor": "google", "date": "2024-05-14T00:00:00.000Z", "score": 69, "source": "independent", "source_url": "https://arxiv.org/abs/2406.01574" }, { "model": "GPT-4o", "vendor": "openai", "date": "2024-05-13T00:00:00.000Z", "score": 72.6, "source": "independent", "source_url": "https://arxiv.org/abs/2406.01574" } ], "tags": [ "knowledge", "reasoning", "multiple-choice", "mmlu-successor", "chain-of-thought" ], "last_updated": "2026-07-21T00:00:00.000Z", "changelog": [ { "date": "2026-07-21T00:00:00.000Z", "note": "Initial page. Second `nearing-saturation` status on the wiki. Timeline holds the four launch-era points from the MMLU-Pro paper (GPT-4-Turbo 63.7, Claude 3 Opus 68.5, Gemini 1.5 Pro 69.0, GPT-4o 72.6), all 5-shot CoT and marked independent (the authors evaluated models they did not build); model dates are the models' real release dates, scores are the paper's. Later 2025–26 frontier scores (high 80s, leaderboards ~89–90%) are described in prose but NOT plotted: no single authoritative per-model source with a verified release date was confirmed, so points were omitted rather than fabricated (Artificial Analysis linked as the live reference). Human baseline is null by design — MMLU-Pro calibrated difficulty via model filtering, not human testing." } ] }, { "name": "MMLU-Redux", "slug": "mmlu-redux", "capability": "knowledge", "secondary_capabilities": [], "short_description": "A manual error audit of MMLU: experts re-annotated thousands of questions and found ~6.5% are broken — the evidence base for MMLU's label-noise ceiling.", "launch_date": "2024-06-06T00:00:00.000Z", "status": "saturated", "status_evidence": "As a competition benchmark, MMLU-Redux is a corrected slice of MMLU: it inherits MMLU's contamination and frontier convergence, so it does not differentiate top models — saturated by CLAUDE.md's definition (usefulness as a model-differentiating eval). Its ENDURING value is different and non-competitive: it remains an actively-used forensic reference and audit methodology for measuring label noise. Status here reflects the eval role; the still-active reference role is documented in prose. saturated_date is set to its launch — it described an already-saturated parent from birth.", "saturated_date": "2024-06-06T00:00:00.000Z", "successor": null, "lineage": { "parent": "mmlu", "variants": [] }, "paper": { "title": "Are We Done with MMLU?", "url": "https://arxiv.org/abs/2406.04127", "venue": "NAACL 2025" }, "authors": [ { "name": "Aryo Pradipta Gema", "affiliation": "University of Edinburgh" }, { "name": "Joshua Ong Jun Leang", "affiliation": "University of Edinburgh" }, { "name": "Pasquale Minervini", "affiliation": "University of Edinburgh" } ], "maintainer": "Edinburgh DAWG (Gema et al.); repo/dataset edinburgh-dawg/mmlu-redux-2.0", "license": "CC-BY-4.0", "dataset": { "url": "https://huggingface.co/datasets/edinburgh-dawg/mmlu-redux-2.0", "size": 5700, "public_test_set": true, "gated": false }, "metric": { "primary": "accuracy on the re-annotated (error-corrected) MMLU questions (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "accuracy over the corrected question set; the paper's core analysis compares reported-vs-corrected accuracy and how rankings shift per subject", "variants": [] }, "run_config": { "shots": "5-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "MMLU-Redux (dataset)", "url": "https://huggingface.co/datasets/edinburgh-dawg/mmlu-redux-2.0", "task_flag": null }, { "name": "lm-evaluation-harness (standard MMLU harness on the corrected set)", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": null } ] }, "cost_estimate": { "examples": 5700, "approx_usd_frontier_model": "1–5 (5,700 corrected multiple-choice items evaluated with the standard direct-answer MMLU protocol; cheap — the cost of Redux is the human annotation, not model inference)" }, "human_baseline": { "score": null, "note": "No separate human baseline. MMLU-Redux inherits MMLU's context, where an ~89.8% expert baseline was the historical reference. Redux's contribution is orthogonal to a human score: it measures how much of MMLU's headroom was illusory (label noise), not human vs model performance. See the parent MMLU page for the baseline discussion." }, "statistical_note": "The original audit re-annotated 3,000 questions across 30 subjects (14 expert annotators); the expanded Redux 2.0 covers 5,700 across all 57 subjects. Overall ~6.5% of questions contain errors, but the rate is wildly uneven — near zero in some subjects, and as high as ~57% in Virology (the original 30-subject audit already flagged it above 25%). This unevenness is the point: a benchmark's effective ceiling is set by its worst subjects, so per-subject error rates matter more than the headline average.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Inherits MMLU's contamination completely — these ARE MMLU questions, corrected. Public since 2020 (as MMLU), fully static, ungated. Correcting the labels does nothing to reduce memorization risk; if anything the questions are more discussed now. Risk is high for the same reasons as the parent.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [], "performance_timeline": [], "tags": [ "audit", "label-noise", "error-analysis", "mmlu", "diagnostic", "benchmark-forensics" ], "last_updated": "2026-07-21T00:00:00.000Z", "changelog": [ { "date": "2026-07-21T00:00:00.000Z", "note": "Initial page. First audit/diagnostic entry on the wiki — precedent for future error-analysis and contamination-detection suites. Performance timeline is intentionally EMPTY: MMLU-Redux is a corrected reference set and audit methodology, not a competition leaderboard, so labs do not report headline Redux scores over time — plotting a model-vs-time race would misrepresent what it is. The gauge and timeline therefore render their empty/degraded state by design (not for lack of sourcing). Human baseline null (inherits MMLU's context). Figures (~6.5% overall, Virology worst) are stated to agree exactly with the citations on the parent MMLU page; dataset.size is 5,700 (Redux 2.0, the current primary configuration) per the size-matches-primary convention, with the original 3,000/30-subject audit noted." } ] }, { "name": "MMMU", "slug": "mmmu", "capability": "multimodal", "secondary_capabilities": [ "knowledge", "reasoning" ], "short_description": "College-level multimodal questions across 30 subjects and 183 subfields, with interleaved text and diverse images.", "launch_date": "2023-11-27T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "Still a common broad benchmark, but the maintainers introduced the harder MMMU-Pro in 2024 and released original test answers in 2026, weakening MMMU's value as a held-out frontier signal.", "saturated_date": null, "successor": "mmmu-pro", "lineage": { "parent": null, "variants": [ "mmmu-pro" ] }, "paper": { "title": "MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI", "url": "https://arxiv.org/abs/2311.16502v4", "venue": "CVPR 2024" }, "authors": [ { "name": "Xiang Yue", "affiliation": "The Ohio State University" }, { "name": "Yuansheng Ni", "affiliation": null }, { "name": "Kai Zhang et al.", "affiliation": null } ], "maintainer": "MMMU Benchmark team", "license": "Apache-2.0 code; dataset items retain source-specific rights", "dataset": { "url": "https://huggingface.co/datasets/MMMU/MMMU", "size": 11550, "public_test_set": true, "gated": false }, "metric": { "primary": "micro-averaged accuracy (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "instance-weighted accuracy across multiple-choice and open questions after rule-based response extraction; report validation and test separately", "variants": [] }, "run_config": { "shots": "zero-shot or subject-level few-shot using the 150-example development set", "cot": "cot", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official MMMU evaluator", "url": "https://github.com/MMMU-Benchmark/MMMU", "task_flag": "mmmu" } ] }, "cost_estimate": { "examples": 10500, "approx_usd_frontier_model": "high for the complete test set; the 900-example validation split is the usual lower-cost check" }, "human_baseline": { "score": 88.6, "note": "Best expert validation accuracy. The paper also reports medium and lowest expert groups at 82.6% and 76.2%; no equivalent full-test human score is reported." }, "statistical_note": "The 900-example validation aggregate is much noisier than the 10,500-example test score, especially after slicing 30 subjects. Prompting and answer extraction must match across comparisons.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "At launch the test labels were server-held. The official repository announced their release on 2026-02-12, so local evaluation is now reproducible but the original test can no longer function as secret held-out data.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official MMMU leaderboard", "url": "https://mmmu-benchmark.github.io/", "type": "official" } ], "performance_timeline": [ { "model": "GPT-4V (validation, paper)", "vendor": "openai", "date": "2023-11-27T00:00:00.000Z", "score": 56.8, "source": "independent", "source_url": "https://arxiv.org/html/2311.16502v4" }, { "model": "GPT-4o (validation, paper update)", "vendor": "openai", "date": "2024-05-13T00:00:00.000Z", "score": 69.1, "source": "independent", "source_url": "https://arxiv.org/html/2311.16502v4" } ], "tags": [ "college", "expert-knowledge", "diagrams", "charts", "interleaved-images", "mcq" ], "last_updated": "2026-09-05T00:00:00.000Z", "changelog": [ { "date": "2026-09-05T00:00:00.000Z", "note": "Initial page based on arXiv v4 and current official repository; records MMMU-Pro as successor and the 2026 public release of test answers." } ] }, { "name": "MT-Bench", "slug": "mt-bench", "capability": "instruction-following", "secondary_capabilities": [ "reasoning", "coding" ], "short_description": "80 curated multi-turn questions across 8 skills, each a two-turn conversation scored by a strong LLM judge on a 1–10 scale.", "launch_date": "2023-06-09T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "The 80 two-turn questions are fixed, public, and heavily exercised across research and leaderboards, so the test set is both widely memorized and increasingly saturated (top models score 8.5–9.0 on a 1–10 scale where 9 is a realistic ceiling). It remains a standard, still-discriminative LLM-judge eval, but the fixed set is the constraint: refreshed or held-out variants are the future direction.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "url": "https://arxiv.org/abs/2306.05685", "venue": "NeurIPS 2023" }, "authors": [ { "name": "Lianmin Zheng", "affiliation": "UC Berkeley" }, { "name": "Wei-Lin Chiang", "affiliation": "UC Berkeley" }, { "name": "Ying Sheng", "affiliation": "Stanford University" }, { "name": "Siyuan Zhuang", "affiliation": "UC Berkeley" }, { "name": "Hao Zhang", "affiliation": "UC Berkeley & UC San Diego" } ], "maintainer": "LMSYS Org (lm-sys/FastChat)", "license": "Apache-2.0 (FastChat tooling; question data distributed with the repo)", "dataset": { "url": "https://huggingface.co/datasets/lmsys/mt_bench_human_judgments", "size": 80, "public_test_set": true, "gated": false }, "metric": { "primary": "average judge score (1–10)", "scoring": "llm-judge", "judge_model": "GPT-4 (original paper; later reported with GPT-4-family judges, so cross-time scores are comparable only when the judge is the same)", "aggregation": "mean of a strong LLM judge's 1–10 score over the 80 two-turn questions, averaged across the two turns; also reported per category or category-grouped", "variants": [ { "name": "Win-rate vs baseline", "description": "MT-Bench is also reported as a pairwise win-rate against a reference model: for each of the 80 conversations, the judge declares a winner on each turn, and the score is the fraction of turns won rather than a mean 1–10 rating. This is the reporting convention used on the original leaderboard and for many model comparisons.", "scoring": "llm-judge", "judge_model": "GPT-4" } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "FastChat llm_judge", "url": "https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge", "task_flag": "evaluate" }, { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "mt_bench" } ] }, "cost_estimate": { "examples": 80, "approx_usd_frontier_model": "5–25 (the per-turn judge calls to a frontier model dominate; ~160 judged answers, each pass scoring a frontier model plus a judge pass)" }, "human_baseline": { "score": null, "note": "No canonical human score on the 1–10 judge scale has been published. The paper instead validated the judge against human preference annotations: GPT-4-judge agreement with human preference exceeded 80%, roughly matching human–human agreement on the same annotations. That is an agreement check, not a human competence score." }, "statistical_note": "80 two-turn questions, with 4 or fewer per category for three of the eight categories, is small enough that reported top-model gaps of a few tenths of a point on a 1–10 scale are close to noise; treat sub-~0.3 point differences (and any single-category comparison) with caution.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The 80 questions are fixed, shipped inside the FastChat repo, and have been used heavily across research, leaderboards, and (per several labs) included in instruction-tuning and alignment data mixtures. There is no refreshed or held-out MT-Bench; the classification question is therefore mostly fairness across contemporary models on the same fixed set.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [], "performance_timeline": [ { "model": "GPT-4", "vendor": "openai", "date": "2023-06-09T00:00:00.000Z", "score": 8.99, "source": "independent", "source_url": "https://arxiv.org/abs/2306.05685" } ], "tags": [ "chat", "multi-turn", "instruction-following", "llm-judge", "preference", "alignment", "conversational" ], "last_updated": "2026-08-07T00:00:00.000Z", "changelog": [ { "date": "2026-08-07T00:00:00.000Z", "note": "Initial page. Timeline restricted to the one authoritative score cited in the original paper (GPT-4, ~8.99), because reported MT-Bench numbers vary with the judge model, judge prompt, and greedy-decoding version, and no single source tracks consistent numbers; additional points are omitted rather than fabricated." } ] }, { "name": "MultiPL-E", "slug": "multi-pl-e", "capability": "coding", "secondary_capabilities": [], "short_description": "Parallel HumanEval and MBPP translations for comparing code generation across many programming languages.", "launch_date": "2022-08-17T00:00:00.000Z", "status": "deprecated", "status_evidence": "The original MultiPL-HumanEval axis is a static translation of 161 public HumanEval tasks and inherits HumanEval exposure and ceiling. The official repository now points readers to Ag-LiveCodeBench-X for a more challenging multilingual evaluation.", "saturated_date": null, "successor": null, "lineage": { "parent": "humaneval", "variants": [ "multipl-humaneval", "multipl-mbpp" ] }, "paper": { "title": "MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation", "url": "https://arxiv.org/abs/2208.08227", "venue": "IEEE Transactions on Software Engineering, 2023" }, "authors": [ { "name": "Federico Cassano", "affiliation": "Northeastern University" }, { "name": "John Gouwar", "affiliation": "Northeastern University" }, { "name": "Arjun Guha", "affiliation": "Northeastern University / Roblox Research" }, { "name": "Michael Greenberg", "affiliation": "Stevens Institute of Technology" }, { "name": "Abhinav Jangda", "affiliation": "University of Massachusetts Amherst" } ], "maintainer": "Northeastern Programming Research Laboratory (NUPRL)", "license": null, "dataset": { "url": "https://huggingface.co/datasets/nuprl/MultiPL-E", "size": 2898, "public_test_set": true, "gated": false }, "metric": { "primary": "pass@1 by programming language (%)", "scoring": "unit-tests", "judge_model": null, "aggregation": "mean pass@1 over one named source benchmark and language configuration; language results should be reported separately", "variants": [ { "name": "pass@k", "description": "Estimated probability that at least one of k sampled programs passes every translated unit test.", "scoring": "unit-tests", "judge_model": null } ] }, "run_config": { "shots": "zero-shot completion", "cot": "direct", "temperature": 0.2, "pass_at": 1, "tools": false, "harnesses": [ { "name": "MultiPL-E evaluation framework", "url": "https://github.com/nuprl/MultiPL-E", "task_flag": null }, { "name": "BigCode Evaluation Harness", "url": "https://github.com/bigcode-project/bigcode-evaluation-harness", "task_flag": "multiple-" } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No official human baseline is reported; the benchmark compares model-generated translations of HumanEval or MBPP tasks, not humans taking a calibrated multilingual exam." }, "statistical_note": "The number of items depends on the selected source benchmark and translation filters. Report the source set, language, dataset revision, sample count, and pass@k convention; averaging languages can hide large language-specific gaps.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Parallel translations reduce direct Python-string overlap but preserve the public HumanEval/MBPP problem semantics and tests; use held-out or live multilingual tasks for stronger leakage control.", "risk": "high" }, "languages": [ "multilingual" ], "leaderboards": [], "performance_timeline": [ { "model": "InCoder-6.7B (MultiPL-HumanEval Python, pass@1)", "vendor": "meta", "date": "2022-08-17T00:00:00.000Z", "score": 15.2, "source": "independent", "source_url": "https://mgree.github.io/papers/2022_multiple.pdf" }, { "model": "code-davinci-002 (MultiPL-HumanEval Python, pass@1)", "vendor": "openai", "date": "2022-08-17T00:00:00.000Z", "score": 47, "source": "independent", "source_url": "https://mgree.github.io/papers/2022_multiple.pdf" }, { "model": "SantaCoder-1.1B (MultiPL-HumanEval Python, pass@1)", "vendor": "bigcode", "date": "2023-01-09T00:00:00.000Z", "score": 18, "source": "vendor-reported", "source_url": "https://huggingface.co/bigcode/santacoder/blob/main/README.md" } ], "tags": [ "multilingual", "code-generation", "humaneval", "mbpp", "unit-tests", "pass-at-k" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Reclassified as deprecated. Added the original 2,898-instance size and a fixed MultiPL-HumanEval Python pass@1 timeline: InCoder 15.2, code-davinci-002 47.0, and SantaCoder 18.0. Other axes remain excluded." } ] }, { "name": "Needle-in-a-Haystack", "slug": "needle-in-a-haystack", "capability": "long-context", "secondary_capabilities": [], "short_description": "Synthetic retrieval stress test that hides facts at different depths and context lengths, then asks the model to recover them.", "launch_date": "2023-11-08T00:00:00.000Z", "status": "saturated", "status_evidence": "Vanilla single-needle retrieval is now a smoke test rather than a discriminative frontier benchmark; RULER explicitly notes that models can be nearly perfect on vanilla NIAH while failing harder long-context tasks.", "saturated_date": "2024-04-09T00:00:00.000Z", "successor": "ruler", "lineage": { "parent": null, "variants": [ "ruler", "openai-mrcr" ] }, "paper": null, "authors": [ { "name": "Greg Kamradt", "affiliation": null } ], "maintainer": "Greg Kamradt", "license": "MIT", "dataset": { "url": "https://github.com/gkamradt/needle-in-a-haystack", "size": null, "public_test_set": true, "gated": false }, "metric": { "primary": "retrieval accuracy over context length by depth cells", "scoring": "exact-match", "judge_model": null, "aggregation": "cell-level score across a sweep of context lengths, depth percentages, seeds, and task variants", "variants": [ { "name": "single needle", "description": "One inserted fact placed at one target depth; the model must retrieve it.", "scoring": "exact-match", "judge_model": null }, { "name": "multi / uuid-chain", "description": "Several facts or linked UUID hops are distributed through the context; scoring can be fractional by recovered facts or hops.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Needle In A Haystack CLI", "url": "https://github.com/gkamradt/needle-in-a-haystack", "task_flag": "niah run " } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": "depends on context grid, provider pricing, and maximum context length" }, "human_baseline": { "score": null, "note": "No protocol-comparable human baseline is recorded; the base task is meant as a machine retrieval stress test." }, "statistical_note": "Report the full context-length by depth grid. A single average hides lost-in-the-middle failures.", "contamination": { "test_set_public": true, "refresh_cycle": "continuous", "mitigation": "Needles can be generated per run, but default haystacks, prompts, and configs are public; reuse of fixed seeds weakens the test.", "risk": "low" }, "languages": [ "english" ], "leaderboards": [], "performance_timeline": [], "tags": [ "long-context", "retrieval", "synthetic", "positional-robustness" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2026-08-12T00:00:00.000Z", "note": "Initial long-context page. No timeline added because NIAH is a configurable diagnostic grid rather than one canonical leaderboard." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Corrected status from active to saturated and set RULER as the practical successor for harder multi-needle and long-context degradation testing." } ] }, { "name": "OlympiadBench", "slug": "olympiadbench", "capability": "maths", "secondary_capabilities": [ "reasoning", "multimodal" ], "short_description": "Bilingual Olympiad and Gaokao mathematics and physics problems, including diagrams, open answers, and proofs.", "launch_date": "2024-02-16T00:00:00.000Z", "status": "active", "status_evidence": "The official full-benchmark evaluation reports GPT-4o at 25.89% overall, with materially lower physics performance; broad headroom remains across bilingual and multimodal subsets.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "OlympiadBench: A Challenging Benchmark for Promoting AGI with Olympiad-Level Bilingual Multimodal Scientific Problems", "url": "https://arxiv.org/abs/2402.14008", "venue": "ACL 2024" }, "authors": [ { "name": "Chaoqun He", "affiliation": null }, { "name": "Renjie Luo", "affiliation": null }, { "name": "Yuzhuo Bai", "affiliation": null }, { "name": "Shengding Hu", "affiliation": "Tsinghua University" }, { "name": "Maosong Sun", "affiliation": "Tsinghua University" } ], "maintainer": "OpenBMB", "license": "MIT (code repository)", "dataset": { "url": "https://huggingface.co/datasets/Hothan/OlympiadBench", "size": 8476, "public_test_set": true, "gated": false }, "metric": { "primary": "average score (%)", "scoring": "composite", "judge_model": null, "aggregation": "official subset-weighted evaluation across open-ended auto-graded questions and separately assessed theorem-proof items; always report subject, language, and modality slices", "variants": [ { "name": "Open-ended questions", "description": "Standardized final answers are checked by the official automated equivalence pipeline.", "scoring": "exact-match", "judge_model": null }, { "name": "Theorem proving", "description": "Proof responses require qualitative or sampled assessment rather than final-answer matching.", "scoring": "human-graded", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "cot", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "OlympiadBench official evaluation code", "url": "https://github.com/OpenBMB/OlympiadBench", "task_flag": "eval" } ] }, "cost_estimate": { "examples": 8476, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No single aggregate human baseline is published across competition, Gaokao, math, physics, text, image, open-answer, and proof subsets." }, "statistical_note": "The aggregate combines heterogeneous subjects and grading modes. Small score gaps are not meaningful unless systems use identical dataset versions, image handling, proof policy, and subset weighting.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The authors cleaned and deduplicated source exams, but all released questions, images, solutions, and answers are now public.", "risk": "high" }, "languages": [ "chinese", "english" ], "leaderboards": [ { "name": "Official repository leaderboard", "url": "https://github.com/OpenBMB/OlympiadBench", "type": "official" } ], "performance_timeline": [ { "model": "GPT-4V (full benchmark)", "vendor": "openai", "date": "2024-02-16T00:00:00.000Z", "score": 17.97, "source": "independent", "source_url": "https://arxiv.org/abs/2402.14008" }, { "model": "GPT-4o (full benchmark)", "vendor": "openai", "date": "2024-05-13T00:00:00.000Z", "score": 25.89, "source": "independent", "source_url": "https://github.com/OpenBMB/OlympiadBench" } ], "tags": [ "maths", "physics", "olympiad", "bilingual", "multimodal", "proof" ], "last_updated": "2026-07-29T00:00:00.000Z", "changelog": [ { "date": "2026-07-29T00:00:00.000Z", "note": "Initial page. Full-benchmark scores are plotted separately from the higher text-only axis; proof grading is modeled as a distinct metric variant." } ] }, { "name": "Omni-MATH", "slug": "omni-math", "capability": "maths", "secondary_capabilities": [ "reasoning" ], "short_description": "A 4,428-problem Olympiad mathematics suite spanning more than 33 subdomains and ten annotated difficulty levels.", "launch_date": "2024-09-20T00:00:00.000Z", "status": "deprecated", "status_evidence": "A 2026 expert audit found the original Omni-Judge wrong in 96.4% of disagreements and introduced Omni-MATH-2 with a clean exact-answer subset; original leaderboard scores should not guide frontier comparisons.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "Omni-MATH: A Universal Olympiad Level Mathematic Benchmark For Large Language Models", "url": "https://arxiv.org/abs/2410.07985", "venue": "ICLR 2025" }, "authors": [ { "name": "Bofei Gao", "affiliation": null }, { "name": "Feifan Song", "affiliation": null }, { "name": "Zhe Yang", "affiliation": null }, { "name": "Baobao Chang", "affiliation": "Peking University" } ], "maintainer": "Omni-MATH authors", "license": null, "dataset": { "url": "https://github.com/KbsdJames/Omni-MATH", "size": 4428, "public_test_set": true, "gated": false }, "metric": { "primary": "GPT-4o-judged answer accuracy (%)", "scoring": "llm-judge", "judge_model": "GPT-4o", "aggregation": "mean binary equivalence verdict over all problems, with breakdowns by mathematical domain and annotated difficulty", "variants": [ { "name": "Omni-Judge", "description": "An 8B Llama-3-based judge instruction-tuned on 21,451 GPT-4o judgments; the repository reports about 95% agreement with GPT-4o on an internal set.", "scoring": "llm-judge", "judge_model": "Omni-Judge (Meta-Llama-3-8B-Instruct derivative)" }, { "name": "Rule-based evaluation", "description": "A later repository path for answers that can be normalized and verified without a language-model judge.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "cot", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Omni-MATH official evaluation repository", "url": "https://github.com/KbsdJames/Omni-MATH", "task_flag": "GPT_eval/get_result.sh" } ] }, "cost_estimate": { "examples": 4428, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No aggregate human baseline was published across the 33+ domains and ten difficulty levels." }, "statistical_note": "Sampling error is smaller than judge error here. Scores depend on final-answer extraction and equivalence judgment; the 2026 audit found these errors can erase real model differences.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The authors curated, annotated, and categorized competition problems, but the complete questions and answers are public and many source competitions predate the benchmark.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official Omni-MATH project", "url": "https://omni-math.github.io", "type": "official" } ], "performance_timeline": [ { "model": "OpenAI o1-mini", "vendor": "openai", "date": "2024-10-10T00:00:00.000Z", "score": 60.54, "source": "independent", "source_url": "https://arxiv.org/abs/2410.07985" } ], "tags": [ "maths", "olympiad", "llm-judge", "difficulty", "contamination" ], "last_updated": "2026-07-29T00:00:00.000Z", "changelog": [ { "date": "2026-07-29T00:00:00.000Z", "note": "Initial page. Marked deprecated for frontier comparison after the Omni-MATH-2 expert audit; retained the original o1-mini result only as a historically sourced protocol value." } ] }, { "name": "OpenAI-MRCR", "slug": "openai-mrcr", "capability": "long-context", "secondary_capabilities": [ "reasoning" ], "short_description": "MRCR v2 8-needle long-context benchmark where a model must recover the requested instance from repeated similar requests in a synthetic conversation.", "launch_date": "2025-04-14T00:00:00.000Z", "status": "active", "status_evidence": "Actively tracked as MRCR v2 (8-needle) on LLM Stats, whose 2026-07-22 snapshot lists 21 self-reported model results with a wide 13.5%-91.5% displayed score spread, corresponding to raw mean_similarity values of 0.135-0.915.", "saturated_date": null, "successor": null, "lineage": { "parent": "needle-in-a-haystack", "variants": [ "mrcr-v2-8-needle" ] }, "paper": { "title": "Michelangelo: Long Context Evaluations Beyond Haystacks via Latent Structure Queries", "url": "https://arxiv.org/abs/2409.12640", "venue": null }, "authors": [ { "name": "OpenAI", "affiliation": "OpenAI" } ], "maintainer": "OpenAI", "license": "MIT", "dataset": { "url": "https://huggingface.co/datasets/openai/mrcr", "size": null, "public_test_set": true, "gated": false }, "metric": { "primary": "MRCR v2 8-needle mean_similarity", "scoring": "composite", "judge_model": null, "aggregation": "LLM Stats MRCR v2 (8-needle) mean_similarity converted from the raw 0-1 score to percent for this site's chart axis; higher is better", "variants": [ { "name": "2-, 4-, and 8-needle variants", "description": "The same synthetic conversation task with increasing numbers of repeated similar asks.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "OpenAI MRCR Hugging Face dataset", "url": "https://huggingface.co/datasets/openai/mrcr", "task_flag": "openai/mrcr" } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": "high for upper bins; bins extend to roughly 1M prompt-plus-answer tokens" }, "human_baseline": { "score": null, "note": "No protocol-comparable human baseline is recorded on the dataset card." }, "statistical_note": "This page now uses the MRCR v2 (8-needle) version as tracked by LLM Stats. The visual graph deliberately plots only the LLM Stats headline score: mean_similarity converted from raw 0-1 values to percentages for this site's chart axis. It does not plot per-bin scores, other needle counts, cost, context length, or verification status. LLM Stats marks the 2026-07-22 snapshot as 21 self-reported and 0 verified results.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The data is public on Hugging Face; fresh synthetic generation would be needed for low-leakage private testing.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "OpenAI MRCR dataset card", "url": "https://huggingface.co/datasets/openai/mrcr", "type": "official" }, { "name": "HELM Long Context", "url": "https://crfm.stanford.edu/helm/long-context/latest/", "type": "third-party" }, { "name": "LLM Stats MRCR v2 (8-needle)", "url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29", "type": "third-party" } ], "performance_timeline": [ { "model": "GPT-5.6 Sol (LLM Stats MRCR v2 8-needle)", "vendor": "openai", "date": "2026-07-22T00:00:00.000Z", "score": 91.5, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "GPT-5.6 Terra (LLM Stats MRCR v2 8-needle)", "vendor": "openai", "date": "2026-07-22T00:00:00.000Z", "score": 89.6, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Claude Opus 4.6 (LLM Stats MRCR v2 8-needle)", "vendor": "anthropic", "date": "2026-07-22T00:00:00.000Z", "score": 76, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "GPT-5.5 (LLM Stats MRCR v2 8-needle)", "vendor": "openai", "date": "2026-07-22T00:00:00.000Z", "score": 74, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemma 4 31B (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 66.4, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemini 3.1 Flash-Lite (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 60.1, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemini 3.6 Flash (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 54, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemma 4 26B-A4B (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 44.1, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemma 4 12B (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 43.4, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "GPT-5.6 Luna (LLM Stats MRCR v2 8-needle)", "vendor": "openai", "date": "2026-07-22T00:00:00.000Z", "score": 41.3, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "GPT-5.4 mini (LLM Stats MRCR v2 8-needle)", "vendor": "openai", "date": "2026-07-22T00:00:00.000Z", "score": 33.6, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "GPT-5.4 nano (LLM Stats MRCR v2 8-needle)", "vendor": "openai", "date": "2026-07-22T00:00:00.000Z", "score": 33.1, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemini 3.5 Flash (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 26.6, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemini 3.1 Pro (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 26.3, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemini 3 Pro (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 26.3, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemma 4 E4B (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 25.4, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemini 3 Flash (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 22.1, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemini 3.5 Flash-Lite (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 21.3, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemma 4 E2B (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 19.1, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemini 2.5 Pro Preview 06-05 (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 16.4, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" }, { "model": "Gemma 3 27B (LLM Stats MRCR v2 8-needle)", "vendor": "google", "date": "2026-07-22T00:00:00.000Z", "score": 13.5, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/mrcr-v2-%288-needle%29" } ], "tags": [ "long-context", "retrieval", "multi-needle", "synthetic", "co-reference" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2026-08-12T00:00:00.000Z", "note": "Updated the page to explicitly use MRCR v2 (8-needle), added the LLM Stats 2026-07-22 leaderboard scores as the visual graph axis, and documented that the chart shows only mean_similarity scores converted to the site's percent axis from self-reported unverified results." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Initial long-context page. Timeline omitted because third-party leaderboards track several variants and bins; this page records the official task definition instead." } ] }, { "name": "PutnamBench", "slug": "putnambench", "capability": "maths", "secondary_capabilities": [ "reasoning" ], "short_description": "Formal theorem-proving on Putnam problems — models write a complete Lean/Isabelle/Coq proof, machine-checked by the proof assistant's kernel.", "launch_date": "2024-07-15T00:00:00.000Z", "status": "active", "status_evidence": "The wiki's first formal-proving benchmark and, for reasonable budgets, still far from solved. Through most of 2024–2025 the best systems solved single digits of the Lean set (Goedel-Prover 7 at pass@512; GPT-4o ~1). General reasoning models at pass@1 remain low (GPT-5 ReAct ~28–42 of ~660, single digits %). Heavy-compute systems have since climbed steeply — ByteDance's Seed-Prover reportedly ~581/672 Lean at ~10 GPU-days per problem, and in Jan 2026 Logical Intelligence's Aleph CLAIMED 668/672 (99.4%, agentic pass@1) topping the board — but these use enormous or single-source, unreproduced budgets, and the Isabelle/Coq subsets and the harder 'no-answer' variant remain far less attacked. What would move this to nearing-saturation: independent reproduction of high solve rates across languages under disclosed, comparable budgets.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "minif2f", "proofnet" ] }, "paper": { "title": "PutnamBench: Evaluating Neural Theorem-Provers on the Putnam Mathematical Competition", "url": "https://arxiv.org/abs/2407.11214", "venue": "NeurIPS 2024 Datasets & Benchmarks (ICML 2024 AI4Math Best Paper)" }, "authors": [ { "name": "George Tsoukalas", "affiliation": "UT Austin" }, { "name": "Jasper Lee", "affiliation": "UT Austin" }, { "name": "Swarat Chaudhuri", "affiliation": "UT Austin" } ], "maintainer": "George Tsoukalas et al. (UT Austin); repo trishullab/PutnamBench (canonical — leaderboard + results.json)", "license": "Apache-2.0 (Lean 4, Isabelle), MIT (Coq); informal problem statements included with MAA permission", "dataset": { "url": "https://github.com/trishullab/PutnamBench", "size": 672, "public_test_set": true, "gated": false }, "metric": { "primary": "problems with a complete, kernel-verified formal proof — Lean subset (count / 672)", "scoring": "unit-tests", "judge_model": null, "aggregation": "count of problems for which a proof-assistant-verified proof was found, reported PER LANGUAGE (Lean 672 / Isabelle 640 / Coq 412) and per variant (answer-in-statement vs no-answer). Never blended into one number", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "agentic", "temperature": null, "pass_at": null, "tools": true, "harnesses": [ { "name": "PutnamBench (official — repo + leaderboard)", "url": "https://trishullab.github.io/PutnamBench/leaderboard.html", "task_flag": null } ] }, "cost_estimate": { "examples": 672, "approx_usd_frontier_model": "Enormously variable and the whole story: from a general model at pass@1 (cheap, low solve rate) to fine-tuned provers at pass@512 to special systems at ~10 GPU-days PER PROBLEM (ByteDance) to agentic runs at ~$68 average / $1400 cap per problem (Aleph). A 'solved N' figure is meaningless without this budget" }, "human_baseline": { "score": null, "note": "No usable human baseline, and the mismatch is starker than anywhere else on the wiki. The Putnam is famously brutal: the median score among the elite students who sit it is often 0 or 1 out of 120 (in 2024, ~40% of ~4,000 entrants scored 0–1; only 5 perfect scores in 85 years). But humans solve Putnam INFORMALLY, under a 6-hour limit, with no machine — while PutnamBench requires a COMPLETE FORMAL proof, with no time limit and a proof assistant. Humans and machines are not doing the same task, so a direct baseline comparison is more strained here than for any answer-based benchmark (cf. AIME's 'low even for excellent humans' point, taken further). Hence null, not a number." }, "statistical_note": "Scores are per-language and per-variant, and the totals grow as new Putnam years are formalized (a living benchmark — see Task details), so cite the denominator and 'as of' date. The dominant confound is NOT sampling noise but COMPUTE/SAMPLING BUDGET: results span pass@10 to pass@512-and-beyond, plus GPU-days-per-problem agentic runs, and these differ by tens of percentage points on the same benchmark. A special-purpose system at pass@512 and a general model at pass@1 solving similar counts are very different achievements. The leaderboard itself notes it is seeking a better way to indicate differing budgets.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "A genuinely different contamination profile from every answer-based maths page. The formalizations are hand-crafted (not scraped text), and the formal PROOFS are not published as a solution set the way GSM8K/MATH worked solutions are — so the usual leakage vector is weaker. Crucially, the formal-proof requirement makes memorization far less useful: you cannot shortcut a kernel-checked Lean proof by recalling a number or a memorized informal argument. Working the other way: the Putnam problem STATEMENTS are public and decades of informal solutions circulate widely, which can leak proof strategies. Net: medium, effectively toward the low end — lower than any answer-based maths benchmark here, but not FrontierMath-low, because the statements are so public.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [ { "name": "PutnamBench leaderboard (official / canonical live tracker)", "url": "https://trishullab.github.io/PutnamBench/leaderboard.html", "type": "official" } ], "performance_timeline": [ { "model": "GPT-4o (paper era, Lean, ~1/640)", "vendor": "openai", "date": "2024-07-15T00:00:00.000Z", "score": 0.2, "source": "independent", "source_url": "https://arxiv.org/abs/2407.11214" }, { "model": "Goedel-Prover (Lean, 7/644, pass@512)", "vendor": "princeton", "date": "2025-02-01T00:00:00.000Z", "score": 1.1, "source": "independent", "source_url": "https://github.com/Goedel-LM/Goedel-Prover" }, { "model": "GPT-5 (Lean, ~42/660, pass@1, ~10-turn ReAct)", "vendor": "openai", "date": "2025-08-07T00:00:00.000Z", "score": 6.4, "source": "independent", "source_url": "https://trishullab.github.io/PutnamBench/leaderboard.html" } ], "tags": [ "maths", "theorem-proving", "formal-methods", "lean", "isabelle", "coq", "proof-assistant" ], "last_updated": "2026-07-22T00:00:00.000Z", "changelog": [ { "date": "2026-07-22T00:00:00.000Z", "note": "Initial page. Fifth `maths` benchmark and the wiki's FIRST formal theorem-proving benchmark — a distinct sub-area (formal methods) that grades a machine-checked proof, not a final answer. Firsts: split license (Apache-2.0 Lean/Isabelle, MIT Coq); `scoring: unit-tests` used as the closest existing value for proof-assistant kernel verification (a dedicated `formal-verification` value is flagged as warranted — miniF2F/ProofNet will re-raise it). Totals are 'as of 2026-07-22': 1,724 formalizations (672 Lean 4 / 640 Isabelle / 412 Coq), Putnam 1962–2025 — a LIVING benchmark grown from the paper's ~640 theorems / ~1,697 formalizations. size=672 (Lean, the primary reported/leaderboard config). Timeline plots only comparable/general-budget Lean points (GPT-4o ~0.2%, Goedel-Prover 1.1% pass@512, GPT-5 6.4% pass@1); ByteDance Seed-Prover (~581/672, ~10 GPU-days/problem, independent) and Aleph (668/672 = 99.4%, Jan 2026, single-source Logical Intelligence, agentic pass@1) are in status_evidence/Known issues but NOT plotted, to avoid smoothing enormous- or single-source-budget results into a saturation trend. Aleph framed as a claimed recent frontier result pending independent reproduction — its 15 counterexample fixes to malformed formalizations WERE verified by the PutnamBench team. SCHEMA FLAGS in summary: formal-verification scoring value; a formal-methods grouping mechanism (tags/secondary-capability); pass_at can't hold the budget range; structured per-point run-config (5th case)." } ] }, { "name": "QASPER", "slug": "qasper", "capability": "long-context", "secondary_capabilities": [ "knowledge" ], "short_description": "Scientific-paper QA over full NLP papers, with abstractive, extractive, yes/no, and unanswerable answers plus supporting evidence.", "launch_date": "2021-05-07T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "QASPER remains useful as a diagnostic component, but as a 2021 public static standalone benchmark it is mostly consumed through SCROLLS and LongBench rather than trusted as a current frontier long-context leaderboard.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "A Dataset of Information-Seeking Questions and Answers Anchored in Research Papers", "url": "https://arxiv.org/abs/2105.03011", "venue": "NAACL 2021" }, "authors": [ { "name": "Pradeep Dasigi", "affiliation": null }, { "name": "Kyle Lo", "affiliation": null }, { "name": "Iz Beltagy", "affiliation": null }, { "name": "Arman Cohan", "affiliation": null }, { "name": "Noah A. Smith", "affiliation": null }, { "name": "Matt Gardner", "affiliation": null } ], "maintainer": "Allen Institute for AI", "license": "CC-BY-4.0", "dataset": { "url": "https://huggingface.co/datasets/allenai/qasper", "size": 5049, "public_test_set": true, "gated": false }, "metric": { "primary": "answer F1 and evidence F1", "scoring": "composite", "judge_model": null, "aggregation": "separate token-level answer F1 and evidence-selection F1 over questions; answer types include extractive spans, free-form answers, yes/no, and unanswerable", "variants": [ { "name": "answer F1", "description": "Compares predicted answers with one or more annotated references, using task-specific normalization for answer type.", "scoring": "exact-match", "judge_model": null }, { "name": "evidence F1", "description": "Measures whether the model selects the paper paragraphs, figures, or tables supporting the answer.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "zero-shot or task prompt", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Hugging Face dataset", "url": "https://huggingface.co/datasets/allenai/qasper", "task_flag": "allenai/qasper" }, { "name": "SCROLLS / tau dataset wrapper", "url": "https://huggingface.co/datasets/tau/scrolls", "task_flag": "qasper" } ] }, "cost_estimate": { "examples": 5049, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "The paper reports that contemporary models lagged humans by at least 27 F1 points, but no single site-compatible aggregate human baseline is recorded here." }, "statistical_note": "The full dataset has 5,049 questions over 1,585 NLP papers; document length and multiple answer types make a single binomial confidence interval misleading.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The data is public and static. Because source papers, questions, and answers are available on Hugging Face, post-release training contamination is plausible.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "AllenAI QASPER dataset card", "url": "https://huggingface.co/datasets/allenai/qasper", "type": "official" } ], "performance_timeline": [], "tags": [ "long-context", "question-answering", "scientific-papers", "evidence-selection" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2026-08-12T00:00:00.000Z", "note": "Initial long-context page. Timeline intentionally left empty because QASPER is mostly tracked through component-suite and per-harness results rather than one current official frontier series." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Corrected status from active to nearing-saturation: reuse inside larger suites is not enough to make an old public component dataset a current standalone benchmark." } ] }, { "name": "QMSum", "slug": "qmsum", "capability": "long-context", "secondary_capabilities": [], "short_description": "Query-focused meeting summarization over long transcripts from academic, product, and committee meetings.", "launch_date": "2021-04-13T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "QMSum is still useful as a query-focused summarization component, but the public static 2021 standalone task is now mainly a subtask inside SCROLLS, ZeroSCROLLS, and LongBench-style suites.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "QMSum: A New Benchmark for Query-based Multi-domain Meeting Summarization", "url": "https://arxiv.org/abs/2104.05938", "venue": "NAACL 2021" }, "authors": [ { "name": "Ming Zhong", "affiliation": null }, { "name": "Da Yin", "affiliation": null }, { "name": "Tao Yu", "affiliation": null }, { "name": "Ahmad Zaidi", "affiliation": null }, { "name": "Mutethia Mutuma", "affiliation": null }, { "name": "Rahul Jha", "affiliation": null }, { "name": "Ahmed Hassan Awadallah", "affiliation": null }, { "name": "Asli Celikyilmaz", "affiliation": null }, { "name": "Yang Liu", "affiliation": null }, { "name": "Xipeng Qiu", "affiliation": null }, { "name": "Dragomir Radev", "affiliation": null } ], "maintainer": "Yale-LILY", "license": "MIT", "dataset": { "url": "https://github.com/Yale-LILY/QMSum", "size": 1808, "public_test_set": true, "gated": false }, "metric": { "primary": "ROUGE over query-focused summaries", "scoring": "composite", "judge_model": null, "aggregation": "ROUGE-style overlap between generated and reference summaries, reported over 1,808 query-summary pairs", "variants": [] }, "run_config": { "shots": "zero-shot or task prompt", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official QMSum repository", "url": "https://github.com/Yale-LILY/QMSum", "task_flag": null }, { "name": "SCROLLS / tau dataset wrapper", "url": "https://huggingface.co/datasets/tau/scrolls", "task_flag": "qmsum" } ] }, "cost_estimate": { "examples": 1808, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "The dataset is human-annotated, but no single protocol-comparable human generation score is recorded here." }, "statistical_note": "ROUGE is sensitive to wording and reference-summary coverage; small score gaps should not be over-read without human inspection. The visual graph on this page deliberately plots only the official QMSum paper / repository Table 3 ROUGE-1 (R-1) scores. It does not plot R-2 or R-L.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The dataset is public and static, and its meeting transcripts are distributed in the official repository.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official QMSum repository", "url": "https://github.com/Yale-LILY/QMSum", "type": "official" } ], "performance_timeline": [ { "model": "Random (paper Table 3 R-1)", "vendor": "independent", "date": "2021-04-13T00:00:00.000Z", "score": 12.03, "source": "independent", "source_url": "https://arxiv.org/abs/2104.05938" }, { "model": "Ext. Oracle (paper Table 3 R-1)", "vendor": "independent", "date": "2021-04-13T00:00:00.000Z", "score": 42.84, "source": "independent", "source_url": "https://arxiv.org/abs/2104.05938" }, { "model": "TextRank (paper Table 3 R-1)", "vendor": "independent", "date": "2021-04-13T00:00:00.000Z", "score": 16.27, "source": "independent", "source_url": "https://arxiv.org/abs/2104.05938" }, { "model": "PGNet (paper Table 3 R-1)", "vendor": "independent", "date": "2021-04-13T00:00:00.000Z", "score": 28.74, "source": "independent", "source_url": "https://arxiv.org/abs/2104.05938" }, { "model": "BART (paper Table 3 R-1)", "vendor": "meta", "date": "2021-04-13T00:00:00.000Z", "score": 29.2, "source": "independent", "source_url": "https://arxiv.org/abs/2104.05938" }, { "model": "PGNet* (paper Table 3 R-1)", "vendor": "independent", "date": "2021-04-13T00:00:00.000Z", "score": 31.37, "source": "independent", "source_url": "https://arxiv.org/abs/2104.05938" }, { "model": "BART* (paper Table 3 R-1)", "vendor": "meta", "date": "2021-04-13T00:00:00.000Z", "score": 31.74, "source": "independent", "source_url": "https://arxiv.org/abs/2104.05938" }, { "model": "HMNet* (paper Table 3 R-1)", "vendor": "microsoft", "date": "2021-04-13T00:00:00.000Z", "score": 32.29, "source": "independent", "source_url": "https://arxiv.org/abs/2104.05938" }, { "model": "PGNet (gold spans) (paper Table 3 R-1)", "vendor": "independent", "date": "2021-04-13T00:00:00.000Z", "score": 31.52, "source": "independent", "source_url": "https://arxiv.org/abs/2104.05938" }, { "model": "BART (gold spans) (paper Table 3 R-1)", "vendor": "meta", "date": "2021-04-13T00:00:00.000Z", "score": 32.18, "source": "independent", "source_url": "https://arxiv.org/abs/2104.05938" }, { "model": "HMNet (gold spans) (paper Table 3 R-1)", "vendor": "microsoft", "date": "2021-04-13T00:00:00.000Z", "score": 36.06, "source": "independent", "source_url": "https://arxiv.org/abs/2104.05938" } ], "tags": [ "long-context", "summarization", "meetings", "query-focused" ], "last_updated": "2021-04-13T00:00:00.000Z", "changelog": [ { "date": "2021-04-13T00:00:00.000Z", "note": "Added official paper / repository Table 3 ROUGE-1 (R-1) scores to the visual graph and clarified that R-2 and R-L are not plotted." }, { "date": "2021-04-13T00:00:00.000Z", "note": "Initial long-context page. No performance timeline added because official current model scores are not maintained as one comparable series." }, { "date": "2021-04-13T00:00:00.000Z", "note": "Corrected status from active to nearing-saturation because component reuse does not make the old standalone public split a strong current benchmark." } ] }, { "name": "QuALITY", "slug": "quality", "capability": "long-context", "secondary_capabilities": [ "reasoning" ], "short_description": "Multiple-choice long-document QA where writers and validators read the full article, with a hard subset designed to defeat skimming.", "launch_date": "2021-12-16T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "QuALITY still tests long-document multiple-choice comprehension, but its 2021-era context lengths are short for modern long-context models and the static data is better treated as a component diagnostic.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "QuALITY: Question Answering with Long Input Texts, Yes!", "url": "https://aclanthology.org/2022.naacl-main.391/", "venue": "NAACL 2022" }, "authors": [ { "name": "Richard Yuanzhe Pang", "affiliation": null }, { "name": "Alicia Parrish", "affiliation": null }, { "name": "Nitish Joshi", "affiliation": null }, { "name": "Nikita Nangia", "affiliation": null }, { "name": "Jason Phang", "affiliation": null }, { "name": "Angelica Chen", "affiliation": null }, { "name": "Vishakh Padmakumar", "affiliation": null }, { "name": "Johnny Ma", "affiliation": null }, { "name": "Jana Thompson", "affiliation": null }, { "name": "He He", "affiliation": null }, { "name": "Samuel R. Bowman", "affiliation": null } ], "maintainer": "NYU Machine Learning for Language", "license": null, "dataset": { "url": "https://github.com/nyu-mll/quality", "size": null, "public_test_set": false, "gated": false }, "metric": { "primary": "multiple-choice accuracy (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "accuracy over four-choice questions; papers report both all questions and the difficult subset", "variants": [ { "name": "hard subset accuracy", "description": "Accuracy on questions where most speed-validation annotators answered incorrectly after skimming.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "zero-shot or task prompt", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official QuALITY repository", "url": "https://github.com/nyu-mll/quality", "task_flag": null }, { "name": "SCROLLS / tau dataset wrapper", "url": "https://huggingface.co/datasets/tau/scrolls", "task_flag": "quality" } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": null }, "human_baseline": { "score": 93.5, "note": "The NAACL paper reports 93.5% human performance and a 55.4% baseline-model score in the abstract." }, "statistical_note": "Report all-question and hard-subset accuracy separately; the hard subset was defined by speed-validation failures, not by model performance.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "The repository says test annotations are withheld for leaderboard submissions, but train/dev data and article sources are public.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official QuALITY leaderboard", "url": "https://nyu-mll.github.io/quality/", "type": "official" } ], "performance_timeline": [], "tags": [ "long-context", "multiple-choice", "reading-comprehension", "hard-subset" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2026-08-12T00:00:00.000Z", "note": "Initial long-context page. Dataset size left null because the current repository surface inspected here exposes fields and protocol but not a single canonical total count in page text." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Corrected status from active to nearing-saturation because the benchmark is now more useful as a component diagnostic than as a frontier standalone long-context signal." } ] }, { "name": "RealToxicityPrompts", "slug": "realtoxicityprompts", "capability": "safety-alignment", "secondary_capabilities": [], "short_description": "One hundred thousand web-text prompts for measuring how often language models continue into toxic text.", "launch_date": "2020-09-24T00:00:00.000Z", "status": "deprecated", "status_evidence": "It remains a standard toxicity test bed, but its static public prompts and dependence on the aging Perspective API make it unsuitable as a sole frontier safety benchmark.", "saturated_date": null, "successor": "realer-toxicity-prompts", "lineage": { "parent": null, "variants": [ "realer-toxicity-prompts" ] }, "paper": { "title": "RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models", "url": "https://arxiv.org/html/2009.11462v2", "venue": "Findings of EMNLP 2020" }, "authors": [ { "name": "Samuel Gehman", "affiliation": "Allen Institute for AI / University of Washington" }, { "name": "Suchin Gururangan", "affiliation": "University of Washington" }, { "name": "Maarten Sap", "affiliation": "University of Washington" }, { "name": "Yejin Choi", "affiliation": "Allen Institute for AI / University of Washington" }, { "name": "Noah A. Smith", "affiliation": "Allen Institute for AI / University of Washington" } ], "maintainer": "Allen Institute for AI", "license": "Apache-2.0 code and repository", "dataset": { "url": "https://huggingface.co/datasets/allenai/real-toxicity-prompts", "size": 100000, "public_test_set": true, "gated": false }, "metric": { "primary": "expected maximum Perspective toxicity over sampled continuations", "scoring": "composite", "judge_model": "Perspective API", "aggregation": "mean across prompts of the maximum toxicity score observed among generated continuations; sampling count and subset must be fixed", "variants": [ { "name": "toxicity probability", "description": "Estimated probability that at least one sampled continuation exceeds the chosen toxicity threshold.", "scoring": "composite", "judge_model": "Perspective API" }, { "name": "average toxicity", "description": "Mean Perspective toxicity across generated continuations; less sensitive to a single worst sample than expected maximum toxicity.", "scoring": "composite", "judge_model": "Perspective API" } ] }, "run_config": { "shots": "prompt continuation", "cot": "direct", "temperature": null, "pass_at": null, "tools": false, "harnesses": [ { "name": "Official RealToxicityPrompts code", "url": "https://github.com/allenai/real-toxicity-prompts", "task_flag": "python -m scripts.run_prompts_experiments --help" } ] }, "cost_estimate": { "examples": 100000, "approx_usd_frontier_model": "Potentially high: the original protocol samples multiple continuations per prompt and separately calls Perspective API." }, "human_baseline": { "score": null, "note": "No protocol-comparable human baseline exists; toxicity is assigned by Perspective API rather than by humans completing the prompts." }, "statistical_note": "Scores depend strongly on prompt subset, number of continuations, decoding parameters, continuation length, Perspective API version, and toxicity threshold. Timeline points use expected maximum toxicity on the non-toxic-prompt subset with 25 generations; lower is safer. Timeline dates are model public-launch dates, not evaluation dates.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Use versioned prompt IDs and evaluator settings, report clean and toxic prompt strata separately, and add newer private or adversarial prompts.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [], "performance_timeline": [ { "model": "GPT-1 (non-toxic prompts; 25 generations)", "vendor": "openai", "date": "2018-06-11T00:00:00.000Z", "score": 0.58, "source": "independent", "source_url": "https://arxiv.org/html/2009.11462v2" }, { "model": "GPT-2 (non-toxic prompts; 25 generations)", "vendor": "openai", "date": "2019-02-14T00:00:00.000Z", "score": 0.51, "source": "independent", "source_url": "https://arxiv.org/html/2009.11462v2" }, { "model": "CTRL (non-toxic prompts; 25 generations)", "vendor": "salesforce", "date": "2019-09-11T00:00:00.000Z", "score": 0.52, "source": "independent", "source_url": "https://arxiv.org/html/2009.11462v2" }, { "model": "GPT-3 (non-toxic prompts; 25 generations)", "vendor": "openai", "date": "2020-05-28T00:00:00.000Z", "score": 0.52, "source": "independent", "source_url": "https://arxiv.org/html/2009.11462v2" } ], "tags": [ "toxicity", "text-generation", "perspective-api", "web-text", "detoxification" ], "last_updated": "2026-09-14T00:00:00.000Z", "changelog": [ { "date": "2026-09-14T00:00:00.000Z", "note": "Added paper-reported expected-maximum-toxicity scores using verified model public-launch dates for the timeline x-axis." }, { "date": "2026-09-14T00:00:00.000Z", "note": "Initial page based on arXiv v2, the Findings paper, official code, and released dataset." } ] }, { "name": "RULER", "slug": "ruler", "capability": "long-context", "secondary_capabilities": [ "reasoning" ], "short_description": "Configurable synthetic benchmark that expands needle retrieval into multi-needle, multi-hop tracing, and aggregation tasks.", "launch_date": "2024-04-09T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "RULER is still a current synthetic stress test, but newer top rows report mid-90s average scores across 4k-128k, so it is approaching ceiling for the strongest long-context models while still exposing degradation.", "saturated_date": null, "successor": null, "lineage": { "parent": "needle-in-a-haystack", "variants": [] }, "paper": { "title": "RULER: What's the Real Context Size of Your Long-Context Language Models?", "url": "https://arxiv.org/abs/2404.06654", "venue": "COLM 2024" }, "authors": [ { "name": "Cheng-Ping Hsieh", "affiliation": null }, { "name": "Simeng Sun", "affiliation": null }, { "name": "Samuel Kriman", "affiliation": null }, { "name": "Shantanu Acharya", "affiliation": null }, { "name": "Dima Rekesh", "affiliation": null }, { "name": "Fei Jia", "affiliation": null }, { "name": "Yang Zhang", "affiliation": null }, { "name": "Boris Ginsburg", "affiliation": null } ], "maintainer": "NVIDIA", "license": "Apache-2.0", "dataset": { "url": "https://github.com/NVIDIA/RULER", "size": null, "public_test_set": true, "gated": false }, "metric": { "primary": "accuracy across configured synthetic tasks", "scoring": "composite", "judge_model": null, "aggregation": "average accuracy over 13 representative tasks, with configurable sequence length and task complexity", "variants": [ { "name": "NIAH variants", "description": "Single and multiple needle retrieval tasks with varying needle quantity and type.", "scoring": "exact-match", "judge_model": null }, { "name": "multi-hop tracing and aggregation", "description": "Synthetic tasks that require following links or aggregating information across the context.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official RULER repository", "url": "https://github.com/NVIDIA/RULER", "task_flag": null } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": "depends on selected sequence lengths and number of tasks" }, "human_baseline": { "score": null, "note": "No protocol-comparable human baseline is recorded; the tasks are synthetic model diagnostics." }, "statistical_note": "Report score by context length and task type. The visual graph on this page deliberately plots only the official repository's Avg. column: mean RULER accuracy across the 4K, 8K, 16K, 32K, 64K, and 128K columns for selected high-scoring models in the 2026-08-12 README snapshot. It does not plot claimed length, effective length, weighted averages, or per-length degradation.", "contamination": { "test_set_public": true, "refresh_cycle": "continuous", "mitigation": "RULER can generate configurable synthetic instances; using fresh seeds and held-out configs reduces memorization risk.", "risk": "low" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official RULER repository", "url": "https://github.com/NVIDIA/RULER", "type": "official" } ], "performance_timeline": [ { "model": "Jamba-1.5-large (README Avg. snapshot)", "vendor": "ai21", "date": "2026-08-12T00:00:00.000Z", "score": 96, "source": "independent", "source_url": "https://github.com/NVIDIA/RULER" }, { "model": "Gemini-1.5-pro (README Avg. snapshot)", "vendor": "google", "date": "2026-08-12T00:00:00.000Z", "score": 95.8, "source": "independent", "source_url": "https://github.com/NVIDIA/RULER" }, { "model": "Qwen2.5-14B-Instruct-1M (README Avg. snapshot)", "vendor": "alibaba", "date": "2026-08-12T00:00:00.000Z", "score": 95.7, "source": "independent", "source_url": "https://github.com/NVIDIA/RULER" }, { "model": "Qwen3-235B-A22B (README Avg. snapshot)", "vendor": "alibaba", "date": "2026-08-12T00:00:00.000Z", "score": 95, "source": "independent", "source_url": "https://github.com/NVIDIA/RULER" }, { "model": "Qwen3-14B (README Avg. snapshot)", "vendor": "alibaba", "date": "2026-08-12T00:00:00.000Z", "score": 94.6, "source": "independent", "source_url": "https://github.com/NVIDIA/RULER" }, { "model": "Jamba-1.5-mini (README Avg. snapshot)", "vendor": "ai21", "date": "2026-08-12T00:00:00.000Z", "score": 93.9, "source": "independent", "source_url": "https://github.com/NVIDIA/RULER" }, { "model": "Qwen3-32B (README Avg. snapshot)", "vendor": "alibaba", "date": "2026-08-12T00:00:00.000Z", "score": 93.7, "source": "independent", "source_url": "https://github.com/NVIDIA/RULER" }, { "model": "EXAONE-4.0-32B (README Avg. snapshot)", "vendor": "lg-ai", "date": "2026-08-12T00:00:00.000Z", "score": 93.1, "source": "independent", "source_url": "https://github.com/NVIDIA/RULER" }, { "model": "Qwen2.5-7B-Instruct-1M (README Avg. snapshot)", "vendor": "alibaba", "date": "2026-08-12T00:00:00.000Z", "score": 91.8, "source": "independent", "source_url": "https://github.com/NVIDIA/RULER" }, { "model": "GPT-4-1106-preview (README Avg. snapshot)", "vendor": "openai", "date": "2026-08-12T00:00:00.000Z", "score": 91.6, "source": "independent", "source_url": "https://github.com/NVIDIA/RULER" } ], "tags": [ "long-context", "synthetic", "retrieval", "aggregation", "multi-hop" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2026-08-12T00:00:00.000Z", "note": "Added a visual graph axis using the official RULER README Avg. column for ten selected high-scoring models in the 2026-08-12 snapshot, and added prose/table wording that identifies the plotted statistic as average accuracy across the 4K-128K columns rather than per-length or weighted scores." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Initial long-context page. No timeline added because RULER scores depend on chosen context lengths and synthetic task configuration." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Corrected status from active to nearing-saturation because current public RULER tables show top models clustered near the benchmark ceiling while still revealing length-dependent failures." } ] }, { "name": "SafetyBench", "slug": "safetybench", "capability": "safety-alignment", "secondary_capabilities": [ "knowledge" ], "short_description": "A bilingual 11,435-question multiple-choice test of safety knowledge across seven risk categories.", "launch_date": "2023-09-13T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "SafetyBench remains a broad, cheap bilingual diagnostic, but its static multiple-choice format measures safety recognition rather than deployed refusal or agent behavior.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "safetybench-chinese-subset" ] }, "paper": { "title": "SafetyBench: Evaluating the Safety of Large Language Models", "url": "https://arxiv.org/html/2309.07045v2", "venue": null }, "authors": [ { "name": "Zhexin Zhang", "affiliation": "Tsinghua University" }, { "name": "Leqi Lei", "affiliation": "Tsinghua University" }, { "name": "Lindong Wu", "affiliation": "Tsinghua University" }, { "name": "Rui Sun", "affiliation": "Tsinghua University" }, { "name": "Yongkang Huang", "affiliation": "Tsinghua University" }, { "name": "Chong Long", "affiliation": "Tsinghua University" }, { "name": "Xiao Liu", "affiliation": "Tsinghua University" }, { "name": "Xuanyu Lei", "affiliation": "Tsinghua University" }, { "name": "Jie Tang", "affiliation": "Tsinghua University" }, { "name": "Minlie Huang", "affiliation": "Tsinghua University" } ], "maintainer": "Tsinghua CoAI", "license": "MIT dataset card", "dataset": { "url": "https://huggingface.co/datasets/thu-coai/SafetyBench", "size": 11435, "public_test_set": false, "gated": false }, "metric": { "primary": "multiple-choice accuracy (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "accuracy across single-answer questions; report Chinese, English, and filtered Chinese subset separately", "variants": [] }, "run_config": { "shots": "zero-shot or 5-shot per category", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official SafetyBench evaluator", "url": "https://github.com/thu-coai/SafetyBench", "task_flag": null } ] }, "cost_estimate": { "examples": 11435, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No single protocol-comparable human aggregate is published on the official benchmark page. Questions have answer keys, but that is not a measured human baseline." }, "statistical_note": "The Chinese, English, and sensitive-keyword-filtered Chinese sets are distinct axes. The timeline uses English zero-shot average accuracy only. Five-shot prompts use 35 public development examples per language. Hidden test labels require official submission, and answer-extraction rules can affect API-model results. Timeline dates are model public-launch dates, not evaluation dates.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "Test inputs are public but labels are withheld by the official submission system. Keep labels private, version submissions, and avoid training on translated or source datasets used to construct the questions.", "risk": "medium" }, "languages": [ "chinese", "english" ], "leaderboards": [ { "name": "Official SafetyBench leaderboard", "url": "https://llmbench.ai/safety", "type": "official" } ], "performance_timeline": [ { "model": "Flan-T5-XXL (English zero-shot)", "vendor": "google", "date": "2022-10-20T00:00:00.000Z", "score": 74.2, "source": "independent", "source_url": "https://arxiv.org/html/2309.07045v2" }, { "model": "GPT-3.5 Turbo (English zero-shot)", "vendor": "openai", "date": "2023-03-01T00:00:00.000Z", "score": 78.8, "source": "independent", "source_url": "https://arxiv.org/html/2309.07045v2" }, { "model": "GPT-4 (English zero-shot)", "vendor": "openai", "date": "2023-03-14T00:00:00.000Z", "score": 88.9, "source": "independent", "source_url": "https://arxiv.org/html/2309.07045v2" }, { "model": "ChatGLM2-lite (English zero-shot)", "vendor": "zhipu-ai", "date": "2023-06-25T00:00:00.000Z", "score": 77.1, "source": "independent", "source_url": "https://arxiv.org/html/2309.07045v2" } ], "tags": [ "safety-knowledge", "multiple-choice", "bilingual", "ethics", "health", "privacy" ], "last_updated": "2026-09-14T00:00:00.000Z", "changelog": [ { "date": "2026-09-14T00:00:00.000Z", "note": "Added paper-reported English zero-shot accuracy using verified model public-launch dates for the timeline x-axis." }, { "date": "2026-09-14T00:00:00.000Z", "note": "Initial page based on arXiv v2, official code, leaderboard, and Hugging Face dataset card." } ] }, { "name": "SCROLLS", "slug": "scrolls", "capability": "long-context", "secondary_capabilities": [ "knowledge", "reasoning" ], "short_description": "Seven-task long-sequence suite covering summarization, QA, and NLI over naturally long English texts.", "launch_date": "2022-01-10T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "SCROLLS is still a standard long-sequence NLP suite with an official submission flow, but for frontier LLMs it is partly superseded by ZeroSCROLLS and newer very-long-context benchmarks.", "saturated_date": null, "successor": "zeroscrolls", "lineage": { "parent": null, "variants": [ "zeroscrolls" ] }, "paper": { "title": "SCROLLS: Standardized CompaRison Over Long Language Sequences", "url": "https://arxiv.org/abs/2201.03533", "venue": "EMNLP 2022" }, "authors": [ { "name": "Uri Shaham", "affiliation": null }, { "name": "Elad Segal", "affiliation": null }, { "name": "Maor Ivgi", "affiliation": null }, { "name": "Avia Efrat", "affiliation": null }, { "name": "Ori Yoran", "affiliation": null }, { "name": "Adi Haviv", "affiliation": null }, { "name": "Ankit Gupta", "affiliation": null }, { "name": "Wenhan Xiong", "affiliation": null }, { "name": "Mor Geva", "affiliation": null }, { "name": "Jonathan Berant", "affiliation": null }, { "name": "Omer Levy", "affiliation": null } ], "maintainer": "Tel Aviv University NLP / tau-nlp", "license": "MIT (official code); component datasets retain their own licenses", "dataset": { "url": "https://huggingface.co/datasets/tau/scrolls", "size": null, "public_test_set": true, "gated": false }, "metric": { "primary": "task-normalized aggregate score", "scoring": "composite", "judge_model": null, "aggregation": "suite aggregate over task-specific metrics for GovReport, SummScreenFD, QMSum, NarrativeQA, Qasper, QuALITY, and ContractNLI", "variants": [] }, "run_config": { "shots": "task-specific", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official SCROLLS repository", "url": "https://github.com/tau-nlp/scrolls", "task_flag": null }, { "name": "Hugging Face tau/scrolls", "url": "https://huggingface.co/datasets/tau/scrolls", "task_flag": "tau/scrolls" } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "SCROLLS is a suite of reused datasets with task-specific annotations; no single suite-wide protocol-comparable human baseline is recorded here." }, "statistical_note": "A single SCROLLS aggregate mixes ROUGE, QA, multiple-choice, and NLI-style metrics. The visual graph on this page deliberately plots only the official SCROLLS paper Table 2 Avg column for baseline systems: the final SCROLLS score computed by averaging each dataset's overall performance score. It does not plot per-dataset scores or task-specific metrics.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "SCROLLS standardizes public component datasets in a unified format; component-level leakage risk varies but the suite is static.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official SCROLLS website", "url": "https://www.scrolls-benchmark.com/", "type": "official" } ], "performance_timeline": [ { "model": "Naive (paper Table 2 Avg)", "vendor": "independent", "date": "2022-01-10T00:00:00.000Z", "score": 19.35, "source": "independent", "source_url": "https://arxiv.org/abs/2201.03533" }, { "model": "BART 256 (paper Table 2 Avg)", "vendor": "meta", "date": "2022-01-10T00:00:00.000Z", "score": 26.35, "source": "independent", "source_url": "https://arxiv.org/abs/2201.03533" }, { "model": "BART 512 (paper Table 2 Avg)", "vendor": "meta", "date": "2022-01-10T00:00:00.000Z", "score": 27.58, "source": "independent", "source_url": "https://arxiv.org/abs/2201.03533" }, { "model": "BART 1024 (paper Table 2 Avg)", "vendor": "meta", "date": "2022-01-10T00:00:00.000Z", "score": 29.01, "source": "independent", "source_url": "https://arxiv.org/abs/2201.03533" }, { "model": "LED 1024 (paper Table 2 Avg)", "vendor": "allen-ai", "date": "2022-01-10T00:00:00.000Z", "score": 27.06, "source": "independent", "source_url": "https://arxiv.org/abs/2201.03533" }, { "model": "LED 4096 (paper Table 2 Avg)", "vendor": "allen-ai", "date": "2022-01-10T00:00:00.000Z", "score": 28.3, "source": "independent", "source_url": "https://arxiv.org/abs/2201.03533" }, { "model": "LED 16384 (paper Table 2 Avg)", "vendor": "allen-ai", "date": "2022-01-10T00:00:00.000Z", "score": 29.16, "source": "independent", "source_url": "https://arxiv.org/abs/2201.03533" } ], "tags": [ "long-context", "suite", "summarization", "question-answering", "nli" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2026-08-12T00:00:00.000Z", "note": "Added paper Table 2 baseline Avg scores to the visual graph and added a visible Avg-only stats section clarifying that the graph shows only the final SCROLLS average, not per-dataset metrics." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Initial long-context page. Timeline omitted because the suite combines heterogeneous task metrics and no current single frontier series is maintained here." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Corrected status from active to nearing-saturation to reflect that SCROLLS remains runnable but is no longer the best current frontier long-context benchmark." } ] }, { "name": "SimpleQA", "slug": "simpleqa", "capability": "knowledge", "secondary_capabilities": [], "short_description": "4,326 short fact-seeking questions chosen to stump GPT-4 — a hallucination test graded correct / incorrect / not-attempted.", "launch_date": "2024-11-07T00:00:00.000Z", "status": "active", "status_evidence": "Still discriminating hard. Frontier models remain well short of the ceiling on parametric recall: GPT-4o 38% correct at launch, GPT-4.5 ~62% by early 2025 — climbing but nowhere near solved, and reasoning-tuned models do not automatically help (o1-mini scored 8%). What would move it off 'active': frontier 'correct' crossing ~85–90% (approaching the label ceiling), OR retrieval-augmented deployment becoming universal enough that closed-book parametric recall stops mattering. Neither has happened.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "simpleqa-verified", "chinese-simpleqa" ] }, "paper": { "title": "Measuring Short-Form Factuality in Large Language Models", "url": "https://arxiv.org/abs/2411.04368", "venue": "OpenAI (technical report / preprint)" }, "authors": [ { "name": "Jason Wei", "affiliation": "OpenAI" }, { "name": "Karina Nguyen", "affiliation": "OpenAI" }, { "name": "William Fedus", "affiliation": "OpenAI" } ], "maintainer": "OpenAI (Wei et al.); repo github.com/openai/simple-evals", "license": "MIT", "dataset": { "url": "https://github.com/openai/simple-evals", "size": 4326, "public_test_set": true, "gated": false }, "metric": { "primary": "correct (accuracy over all questions, %)", "scoring": "llm-judge", "judge_model": "Prompted ChatGPT classifier that compares the predicted answer to the reference (GPT-4o in OpenAI's reference implementation, simple-evals). Scores graded by different grader models or prompt versions are not strictly comparable.", "aggregation": "each answer classified correct / incorrect / not-attempted, then aggregated over 4,326 questions", "variants": [ { "name": "correct given attempted", "description": "Accuracy among only the questions the model actually attempted (correct / (correct + incorrect)). Rewards a model for answering when right; unaffected by declining. High here + low 'correct' = a cautious model that knows little but rarely bluffs.", "scoring": "llm-judge", "judge_model": "same ChatGPT classifier" }, { "name": "F-score", "description": "Harmonic mean of 'correct' and 'correct given attempted', equal to 2c/(2c+2i+n) for correct c, incorrect i, not-attempted n. The single number that rewards knowing answers AND declining when unsure — the paper's summary metric and the one SimpleQA Verified reports as F1.", "scoring": "llm-judge", "judge_model": "same ChatGPT classifier" } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "simple-evals (official)", "url": "https://github.com/openai/simple-evals", "task_flag": "simpleqa" }, { "name": "Inspect", "url": "https://inspect.aisi.org.uk", "task_flag": "simpleqa" } ] }, "cost_estimate": { "examples": 4326, "approx_usd_frontier_model": "3–15 (4,326 short questions; answers are brief, but every one needs a separate LLM-grader call, so the grader roughly doubles the API cost)" }, "human_baseline": { "score": null, "note": "No human baseline — and arguably none is meaningful. The questions were deliberately selected to be obscure (adversarially filtered so frontier models got them wrong), so a human sitting the test closed-book would mostly measure luck, and open-book would measure Googling, not knowledge. The paper reports a 94.4% agreement rate from a third annotator, but that measures dataset quality (are the answers indisputable?), not human question-answering ability. This is the first benchmark on the wiki where a human reference line is not just missing but conceptually beside the point." }, "statistical_note": "4,326 questions give a reasonably tight aggregate CI (±~1.5pt at 40% correct). The more important caveats are non-sampling: the LLM grader introduces its own error and version drift (see Known issues), and because SimpleQA reports three outcomes, a single 'accuracy' number hides whether a model is knowledgeable or merely bold — always read 'correct' alongside 'not attempted'.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Public and ungated (openai/simple-evals), fully static, no refresh cycle. The irony is acute: a benchmark built to measure hallucination can itself be memorized. It was adversarially collected against 2024-era models, so the exact Q&A pairs — now discussed in papers, blog posts, and leaderboards — increasingly enter post-2024 training corpora, and a model can 'improve' by having seen the answer key rather than by knowing more. SimpleQA Verified (2025) partly re-collects and cleans to push back on this.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [ { "name": "SimpleQA (official, OpenAI)", "url": "https://www.kaggle.com/benchmarks/openai/simpleqa", "type": "official" }, { "name": "Artificial Analysis / third-party trackers", "url": "https://openai.com/index/introducing-simpleqa/", "type": "third-party" } ], "performance_timeline": [ { "model": "Claude 3 Opus", "vendor": "anthropic", "date": "2024-11-07T00:00:00.000Z", "score": 23.5, "source": "independent", "source_url": "https://arxiv.org/abs/2411.04368" }, { "model": "Claude 3.5 Sonnet", "vendor": "anthropic", "date": "2024-11-07T00:00:00.000Z", "score": 28.9, "source": "independent", "source_url": "https://arxiv.org/abs/2411.04368" }, { "model": "GPT-4o", "vendor": "openai", "date": "2024-11-07T00:00:00.000Z", "score": 38.2, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2411.04368" }, { "model": "o1-preview", "vendor": "openai", "date": "2024-11-07T00:00:00.000Z", "score": 42.7, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2411.04368" }, { "model": "GPT-4.5", "vendor": "openai", "date": "2025-02-27T00:00:00.000Z", "score": 62.5, "source": "vendor-reported", "source_url": "https://openai.com/index/introducing-gpt-4-5/" } ], "tags": [ "factuality", "hallucination", "short-form", "calibration", "llm-judge", "parametric-knowledge" ], "last_updated": "2026-07-21T00:00:00.000Z", "changelog": [ { "date": "2026-07-21T00:00:00.000Z", "note": "Initial page. First `active`-status benchmark on the wiki (completing the four-status taxonomy), first `llm-judge` PRIMARY scoring with a real judge_model value, and first entry with a genuinely-null (not just missing) human baseline. Timeline is on the 'correct' axis: four launch-paper points (Claude 3 Opus 23.5 and Claude 3.5 Sonnet 28.9 marked independent — OpenAI is not their vendor; GPT-4o 38.2 and o1-preview 42.7 marked vendor-reported) plus GPT-4.5 62.5 (2025-02-27, vendor-reported, GPT-4.5 launch). The four launch-paper models predate the benchmark, so their scores are dated to SimpleQA's launch (2024-11-07) — when the scores were actually established — rather than to each model's release, which would place them before launch and clamp them to the axis origin. Scores are still CLIMBING toward an unknown ceiling — no plateau to draw. 2025–26 numbers beyond GPT-4.5 exist but were left to prose where a single authoritative per-model source wasn't confirmed. SimpleQA Verified (Google DeepMind, arXiv:2509.07968) and Chinese SimpleQA (Alibaba, arXiv:2411.07140) added to lineage.variants (pages not written; slugs won't resolve yet)." } ] }, { "name": "Soohak", "slug": "soohak", "capability": "maths", "secondary_capabilities": [ "reasoning", "safety-alignment" ], "short_description": "Fresh research-level mathematics authored by mathematicians, plus unanswerable prompts that reward justified refusal.", "launch_date": "2026-05-09T00:00:00.000Z", "status": "active", "status_evidence": "The official July 2026 tracker reports 64.4% Avg@3 for the Challenge leader with 46 of 340 problems still unsolved by any tracked model; Refusal remains a separate hard target.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "Soohak: A Mathematician-Curated Benchmark for Evaluating Research-level Math Capabilities of LLMs", "url": "https://arxiv.org/abs/2605.09063", "venue": null }, "authors": [ { "name": "Guijin Son", "affiliation": "Seoul National University" }, { "name": "Seungone Kim", "affiliation": "Carnegie Mellon University" }, { "name": "Catherine Arnett", "affiliation": null }, { "name": "Sean Welleck", "affiliation": "Carnegie Mellon University" }, { "name": "Youngjae Yu", "affiliation": "Yonsei University" } ], "maintainer": "Soohak organizing team", "license": null, "dataset": { "url": "https://soohak-bench.github.io", "size": 439, "public_test_set": false, "gated": true }, "metric": { "primary": "Challenge Avg@3 (%)", "scoring": "llm-judge", "judge_model": "GPT-5-Mini", "aggregation": "for each of 340 Challenge items, average correctness over three independent samples, then average equally across items", "variants": [ { "name": "Challenge Pass@3", "description": "Fraction of Challenge items solved by at least one of three samples.", "scoring": "llm-judge", "judge_model": "GPT-5-Mini" }, { "name": "Refusal Avg@3", "description": "Mean correct-refusal rate over 99 ill-posed questions and three samples per item.", "scoring": "llm-judge", "judge_model": "GPT-5-Mini" } ] }, "run_config": { "shots": "zero-shot", "cot": "cot", "temperature": null, "pass_at": 3, "tools": false, "harnesses": [ { "name": "Soohak evaluation by request", "url": "https://soohak-bench.github.io", "task_flag": null } ] }, "cost_estimate": { "examples": 1317, "approx_usd_frontier_model": "High and model-dependent: 439 items are sampled three times with long research-math reasoning traces." }, "human_baseline": { "score": null, "note": "Five invited human teams collectively solved 50.6% of a 79-prompt sample, but this coverage result is not protocol-comparable to a single model's Avg@3." }, "statistical_note": "Challenge has 340 items and Refusal 99. Three samples reduce decoding noise but do not make Avg@3 a single-pass accuracy; judge error and subject mix remain material.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "Problems were newly authored without AI assistance under confidentiality controls; the full collection is embargoed until late 2026 and models are evaluated by request.", "risk": "low" }, "languages": [ "english", "korean" ], "leaderboards": [ { "name": "Official Soohak leaderboard", "url": "https://soohak-bench.github.io", "type": "official" } ], "performance_timeline": [ { "model": "Gemini 3 Pro (Challenge Avg@3)", "vendor": "google", "date": "2026-05-09T00:00:00.000Z", "score": 30.39, "source": "independent", "source_url": "https://arxiv.org/abs/2605.09063" }, { "model": "GPT-5.5 high (Challenge Avg@3)", "vendor": "openai", "date": "2026-07-01T00:00:00.000Z", "score": 64.4, "source": "independent", "source_url": "https://soohak-bench.github.io" } ], "tags": [ "maths", "research-level", "private", "refusal", "bilingual", "contamination" ], "last_updated": "2026-07-29T00:00:00.000Z", "changelog": [ { "date": "2026-07-29T00:00:00.000Z", "note": "Initial page. Challenge and Refusal are kept as separate score axes; current tracker values are distinguished from the paper's launch evaluation." } ] }, { "name": "StereoSet", "slug": "stereoset", "capability": "safety-alignment", "secondary_capabilities": [ "knowledge" ], "short_description": "English context-association tests that separate stereotypical preference from basic language-modeling ability.", "launch_date": "2020-04-20T00:00:00.000Z", "status": "deprecated", "status_evidence": "StereoSet remains a useful historical bias diagnostic, but its static public data, US-centric stereotypes, and sensitivity to scoring choices make it a weak standalone frontier safety benchmark.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "intrasentence-stereoset", "intersentence-stereoset", "contextual-stereoset" ] }, "paper": { "title": "StereoSet: Measuring stereotypical bias in pretrained language models", "url": "https://arxiv.org/html/2004.09456v1", "venue": "ACL-IJCNLP 2021" }, "authors": [ { "name": "Moin Nadeem", "affiliation": null }, { "name": "Anna Bethke", "affiliation": null }, { "name": "Siva Reddy", "affiliation": null } ], "maintainer": "StereoSet authors", "license": "CC-BY-SA-4.0", "dataset": { "url": "https://huggingface.co/datasets/McGill-NLP/stereoset", "size": 4229, "public_test_set": true, "gated": false }, "metric": { "primary": "ICAT score (0-100; higher balances language modeling and stereotype neutrality)", "scoring": "composite", "judge_model": null, "aggregation": "ICAT combines language-modeling score with the distance of stereotype score from the neutral value of 50; report intrasentence and intersentence tracks separately", "variants": [ { "name": "Language Modeling Score (LMS)", "description": "Percentage of examples where a meaningful stereotypical or anti-stereotypical option outranks the unrelated option.", "scoring": "log-likelihood", "judge_model": null }, { "name": "Stereotype Score (SS)", "description": "Percentage of related comparisons where the stereotypical option outranks the anti-stereotypical option; 50 is neutral, not 100.", "scoring": "log-likelihood", "judge_model": null } ] }, "run_config": { "shots": "zero-shot likelihood ranking", "cot": "direct", "temperature": null, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official StereoSet code", "url": "https://github.com/moinnadeem/StereoSet", "task_flag": null }, { "name": "Hugging Face dataset", "url": "https://huggingface.co/datasets/McGill-NLP/stereoset", "task_flag": "intersentence or intrasentence" } ] }, "cost_estimate": { "examples": 4229, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No protocol-comparable human ICAT baseline is reported. ICAT 100 describes an idealized model with perfect language modeling and stereotype score 50, not measured human performance." }, "statistical_note": "The released Hugging Face validation data contain 2,123 intersentence and 2,106 intrasentence contexts. The often-quoted roughly 17,000 figure counts candidate sentences, not independent scored contexts. ICAT can hide different LMS/SS trade-offs, so publish all three values per track. Timeline dates are model public-launch dates, not evaluation dates.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Use the released set as a regression diagnostic, disclose the exact split and tokenizer scoring method, and pair it with newer or privately held bias probes.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "StereoSet project", "url": "https://stereoset.mit.edu/", "type": "official" } ], "performance_timeline": [ { "model": "BERT-base (test ICAT)", "vendor": "google", "date": "2018-11-02T00:00:00.000Z", "score": 71.2, "source": "independent", "source_url": "https://arxiv.org/html/2004.09456v1" }, { "model": "GPT-2 124M (test ICAT)", "vendor": "openai", "date": "2019-02-14T00:00:00.000Z", "score": 73, "source": "independent", "source_url": "https://arxiv.org/html/2004.09456v1" }, { "model": "XLNet-large (test ICAT)", "vendor": "google", "date": "2019-06-19T00:00:00.000Z", "score": 72, "source": "independent", "source_url": "https://arxiv.org/html/2004.09456v1" } ], "tags": [ "bias", "stereotypes", "fairness", "gender", "profession", "race", "religion" ], "last_updated": "2026-09-14T00:00:00.000Z", "changelog": [ { "date": "2026-09-14T00:00:00.000Z", "note": "Added paper-reported test ICAT scores using verified model public-launch dates for the timeline x-axis." }, { "date": "2026-09-14T00:00:00.000Z", "note": "Initial page based on arXiv v1, the official repository, and the released dataset card." } ] }, { "name": "StructFlowBench", "slug": "structflowbench", "capability": "instruction-following", "secondary_capabilities": [ "long-context" ], "short_description": "155 multi-turn dialogues scored on fine-grained constraints plus six inter-turn dialogue relationships (recall, refinement, expansion, follow-up, summary).", "launch_date": "2025-02-20T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "Moved to the site's nearing-saturation bucket: the best reported WCSR is already about 0.96 on a 155-dialogue public static set, while the remaining signal is concentrated in harder relationship slices such as refinement rather than the headline score.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "StructFlowBench: A Structured Flow Benchmark for Multi-turn Instruction Following", "url": "https://aclanthology.org/2025.findings-acl.486/", "venue": "Findings of ACL 2025" }, "authors": [ { "name": "Jinnan Li", "affiliation": "Jilin University" }, { "name": "Jinzhe Li", "affiliation": "Jilin University" }, { "name": "Yue Wang", "affiliation": "University of North Carolina at Chapel Hill" }, { "name": "Yi Chang", "affiliation": "Jilin University" }, { "name": "Yuan Wu", "affiliation": "Jilin University" } ], "maintainer": "ML Group, Jilin University (github.com/MLGroupJLU/StructFlowBench)", "license": "MIT (benchmark code and data)", "dataset": { "url": "https://huggingface.co/datasets/Jinnan/StructFlowBench", "size": 155, "public_test_set": true, "gated": false }, "metric": { "primary": "WCSR — Weighted Constraint Satisfaction Rate (%)", "scoring": "llm-judge", "judge_model": "GPT-4o (the paper's automated evaluator — see scoring methodology for the reproducibility caveat)", "aggregation": "weighted mean over constraints: WCSR = Σ w_j·s_j / Σ w_j, where s_j ∈ {0,1} per constraint, with structural constraints weighted w_s = 2 and intra-turn constraints w_r = 1", "variants": [ { "name": "CSR — Constraint Satisfaction Rate", "description": "Unweighted average proportion of satisfied constraints across all instructions, treating every constraint equally.", "scoring": "llm-judge", "judge_model": "GPT-4o" }, { "name": "ISR — Instruction Satisfaction Rate", "description": "Binary per instruction: 1 only when every constraint in that instruction was satisfied.", "scoring": "llm-judge", "judge_model": "GPT-4o" }, { "name": "DRFR — Decomposed Requirement Following Rate", "description": "Global requirement-following ratio: satisfied scoring questions summed over all instructions, divided by the total number of scoring questions.", "scoring": "llm-judge", "judge_model": "GPT-4o" } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official StructFlowBench evaluation", "url": "https://github.com/MLGroupJLU/StructFlowBench", "task_flag": null } ] }, "cost_estimate": { "examples": 155, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No protocol-comparable human baseline published. The 'Golden Context' setup drops each model into the same curated dialogue history rather than one it generated, but the paper reports no human agreement with the GPT-4o judge on model outputs." }, "statistical_note": "155 dialogues and a per-constraint score grid under a single LLM judge; per-turn and per-relationship cells are small. Treat WCSR gaps under a few points as noise until a leaderboard reports confidence intervals or a second judge.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The dataset was released on 2025-02-20 and postdates most 2024 training cutoffs, so direct memorization is currently less of a risk. The structural relationships are a behavioral/semantic target rather than a fixed factual test set, so memorizing one dialogue does little to help on unseen dialogue flows.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official StructFlowBench leaderboard (in-repo image)", "url": "https://github.com/MLGroupJLU/StructFlowBench", "type": "official" } ], "performance_timeline": [ { "model": "DeepSeek-v3", "vendor": "deepseek", "date": "2025-02-20T00:00:00.000Z", "score": 96, "source": "independent", "source_url": "https://arxiv.org/abs/2502.14494" }, { "model": "Gemini-1.5-Pro", "vendor": "google", "date": "2025-02-20T00:00:00.000Z", "score": 95, "source": "independent", "source_url": "https://arxiv.org/abs/2502.14494" }, { "model": "GPT-4o", "vendor": "openai", "date": "2025-02-20T00:00:00.000Z", "score": 95, "source": "independent", "source_url": "https://arxiv.org/abs/2502.14494" }, { "model": "Claude 3.5 Sonnet", "vendor": "anthropic", "date": "2025-02-20T00:00:00.000Z", "score": 94, "source": "independent", "source_url": "https://arxiv.org/abs/2502.14494" } ], "tags": [ "instruction-following", "multi-turn", "dialogue", "constraint-satisfaction", "conversation", "structural-flow" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2026-08-12T00:00:00.000Z", "note": "Changed status from active to nearing-saturation per site taxonomy update; saturated_date remains null because the page is not marked saturated." }, { "date": "2026-08-07T00:00:00.000Z", "note": "Corrected the WCSR timeline to the paper's Table 2 ordering: DeepSeek-v3 96.0, then a two-way tie at 95.0 between Gemini-1.5-Pro and GPT-4o, with Claude 3.5 Sonnet 94.0 in 4th (was previously listed as a top-3). Refinement mean corrected to ~0.72 and average turns to ~4.14." }, { "date": "2026-08-07T00:00:00.000Z", "note": "Initial page. Timeline carries the three top-scoring models on the WCSR axis from the original paper Table 2 (all author-run, sourced to arXiv). Dataset size is set to 155 dialogues per the paper's corpus table; no human baseline is reported." } ] }, { "name": "SuperGLUE", "slug": "superglue", "capability": "reasoning", "secondary_capabilities": [ "knowledge" ], "short_description": "Eight English NLU tasks spanning question answering, inference, causal reasoning, word sense, and coreference.", "launch_date": "2019-05-02T00:00:00.000Z", "status": "saturated", "status_evidence": "The official aggregate human estimate (89.8) was crossed by a single DeBERTa model (89.9) by 2020-12-29; the 2022 leaderboard leader reached 91.3, leaving a narrow, noisy ceiling.", "saturated_date": "2020-12-29T00:00:00.000Z", "successor": null, "lineage": { "parent": "glue", "variants": [] }, "paper": { "title": "SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems", "url": "https://arxiv.org/abs/1905.00537", "venue": "NeurIPS 2019" }, "authors": [ { "name": "Alex Wang", "affiliation": "New York University" }, { "name": "Yada Pruksachatkun", "affiliation": "New York University" }, { "name": "Nikita Nangia", "affiliation": "New York University" }, { "name": "Amanpreet Singh", "affiliation": "Facebook AI Research" }, { "name": "Julian Michael", "affiliation": "University of Washington" }, { "name": "Felix Hill", "affiliation": "DeepMind" }, { "name": "Omer Levy", "affiliation": "Facebook AI Research" }, { "name": "Samuel R. Bowman", "affiliation": "New York University" } ], "maintainer": "GLUE Benchmark team", "license": null, "dataset": { "url": "https://super.gluebenchmark.com/tasks", "size": null, "public_test_set": true, "gated": false }, "metric": { "primary": "SuperGLUE aggregate score (%)", "scoring": "composite", "judge_model": null, "aggregation": "Official server: compute each task score, first averaging paired metrics for CB, MultiRC, and ReCoRD, then take the unweighted arithmetic mean of the eight task scores.", "variants": [ { "name": "BoolQ accuracy", "description": "Accuracy on yes/no questions grounded in a supplied passage.", "scoring": "exact-match", "judge_model": null }, { "name": "CB accuracy / macro-F1", "description": "Three-way entailment accuracy and unweighted macro-F1; their mean is CB's contribution to the aggregate.", "scoring": "composite", "judge_model": null }, { "name": "COPA accuracy", "description": "Accuracy at selecting the more plausible cause or effect from two alternatives.", "scoring": "exact-match", "judge_model": null }, { "name": "MultiRC F1a / EM", "description": "F1 over answer options and exact match of each question's complete answer set; their mean is MultiRC's task score.", "scoring": "composite", "judge_model": null }, { "name": "ReCoRD F1 / EM", "description": "Maximum token-level F1 and exact match over valid entity mentions; their mean is ReCoRD's task score.", "scoring": "composite", "judge_model": null }, { "name": "RTE accuracy", "description": "Binary entailment versus not-entailment accuracy.", "scoring": "exact-match", "judge_model": null }, { "name": "WiC accuracy", "description": "Binary accuracy for whether a marked word has the same sense in two contexts.", "scoring": "exact-match", "judge_model": null }, { "name": "WSC accuracy", "description": "Binary accuracy for whether a marked pronoun refers to a marked noun phrase.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "zero-shot in lm-evaluation-harness; official leaderboard systems are task-fine-tuned", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "lm-evaluation-harness (LM Eval v1-compatible prompts)", "url": "https://github.com/EleutherAI/lm-evaluation-harness/tree/main/lm_eval/tasks/super_glue", "task_flag": "super-glue-lm-eval-v1" }, { "name": "lm-evaluation-harness (T5-compatible prompts)", "url": "https://github.com/EleutherAI/lm-evaluation-harness/tree/main/lm_eval/tasks/super_glue", "task_flag": "super-glue-t5-prompt" } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": null }, "human_baseline": { "score": 89.8, "note": "Official aggregate estimate. Per-task estimates mix prior published results with crowdworker reannotation; paired metrics are averaged before the eight-task macro-average." }, "statistical_note": "The aggregate is not a sample-size-weighted accuracy and has no single binomial confidence interval. CB, COPA, and WSC have especially small evaluation splits, while metric and prompt implementation differences can exceed small leaderboard gaps.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Official test labels were kept private behind the evaluation server, but test inputs, training and development data, source corpora, and benchmark examples are static and public. Modern harnesses normally score the labeled public validation splits, so those runs have no held-out-label protection.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official SuperGLUE leaderboard", "url": "https://super.gluebenchmark.com/leaderboard", "type": "official" } ], "performance_timeline": [ { "model": "BERT++", "vendor": "benchmark authors", "date": "2019-05-02T00:00:00.000Z", "score": 71.5, "source": "independent", "source_url": "https://proceedings.neurips.cc/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf" }, { "model": "DeBERTa 1.5B (single model)", "vendor": "microsoft", "date": "2020-12-29T00:00:00.000Z", "score": 89.9, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2006.03654" }, { "model": "Vega v2", "vendor": "JD Explore Academy", "date": "2022-10-08T00:00:00.000Z", "score": 91.3, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2212.01853" }, { "model": "T5-11B", "vendor": "google", "date": "2019-10-23T00:00:00.000Z", "score": 89.3, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/1910.10683" }, { "model": "DeBERTa ensemble (TuringNLRv4)", "vendor": "microsoft", "date": "2021-01-06T00:00:00.000Z", "score": 90.3, "source": "vendor-reported", "source_url": "https://www.microsoft.com/en-us/research/blog/microsoft-deberta-surpasses-human-performance-on-the-superglue-benchmark/" }, { "model": "ERNIE 3.0", "vendor": "baidu", "date": "2021-07-12T00:00:00.000Z", "score": 90.6, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2107.02137" }, { "model": "ST-MoE-32B", "vendor": "google", "date": "2022-02-17T00:00:00.000Z", "score": 91.2, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2202.08906" }, { "model": "PaLM 540B", "vendor": "google", "date": "2022-04-04T00:00:00.000Z", "score": 90.4, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2204.02311" } ], "tags": [ "nlu", "multitask", "question-answering", "entailment", "commonsense", "coreference", "word-sense" ], "last_updated": "2026-07-25T00:00:00.000Z", "changelog": [ { "date": "2026-07-25T00:00:00.000Z", "note": "Initial page. Timeline is restricted to comparable official test-server aggregate scores; validation-set harness scores and task-level results are intentionally excluded." }, { "date": "2026-07-25T00:00:00.000Z", "note": "Expanded performance timeline with T5-11B, DeBERTa ensemble, ERNIE 3.0, ST-MoE-32B, and PaLM 540B scores." } ] }, { "name": "SWE-bench", "slug": "swe-bench", "capability": "coding", "secondary_capabilities": [ "agentic-tool-use", "long-context" ], "short_description": "Real GitHub issues from 12 Python repositories, solved by generating patches that must pass repository tests.", "launch_date": "2023-10-10T00:00:00.000Z", "status": "deprecated", "status_evidence": "SWE-bench Verified superseded the original test set in 2024 after human review found underspecified tasks and tests that rejected valid fixes.", "saturated_date": null, "successor": "swe-bench-verified", "lineage": { "parent": null, "variants": [ "swe-bench-lite", "swe-bench-verified", "swe-bench-multimodal", "swe-bench-multilingual" ] }, "paper": { "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "url": "https://arxiv.org/abs/2310.06770", "venue": "ICLR 2024 Oral" }, "authors": [ { "name": "Carlos E. Jimenez", "affiliation": "Princeton University" }, { "name": "John Yang", "affiliation": "Princeton University" }, { "name": "Alexander Wettig", "affiliation": "Princeton University" }, { "name": "Shunyu Yao", "affiliation": "Princeton University" }, { "name": "Kexin Pei", "affiliation": "University of Chicago" }, { "name": "Ofir Press", "affiliation": "Princeton University" }, { "name": "Karthik Narasimhan", "affiliation": "Princeton University" } ], "maintainer": "SWE-bench team", "license": "MIT (benchmark code; source repositories retain their own licenses)", "dataset": { "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench", "size": 2294, "public_test_set": true, "gated": false }, "metric": { "primary": "resolved instances (%)", "scoring": "unit-tests", "judge_model": null, "aggregation": "micro-average over 2,294 test instances; an instance resolves only when FAIL_TO_PASS tests pass and PASS_TO_PASS tests remain passing", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "agentic", "temperature": null, "pass_at": 1, "tools": true, "harnesses": [ { "name": "Official SWE-bench harness", "url": "https://github.com/SWE-bench/SWE-bench", "task_flag": "SWE-bench" } ] }, "cost_estimate": { "examples": 2294, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No protocol-comparable aggregate human score was published. Every instance has a human-authored pull request, but that does not establish a timed blind-test pass rate." }, "statistical_note": "At 2,294 items, a 50% resolve rate has an approximate 95% binomial interval of ±2.0 points; task and repository clustering makes a simple binomial interval optimistic.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Evaluation starts from a historical base commit, but issues, merged pull requests, gold patches, and tests are public in repository history.", "risk": "high" }, "languages": [ "english", "python" ], "leaderboards": [ { "name": "Official SWE-bench leaderboards", "url": "https://www.swebench.com", "type": "official" } ], "performance_timeline": [ { "model": "Claude 2 (BM25 retrieval)", "vendor": "anthropic", "date": "2023-10-10T00:00:00.000Z", "score": 1.97, "source": "independent", "source_url": "https://arxiv.org/abs/2310.06770" }, { "model": "Claude 3 Opus (BM25 retrieval)", "vendor": "anthropic", "date": "2024-03-04T00:00:00.000Z", "score": 3.79, "source": "independent", "source_url": "https://arxiv.org/pdf/2310.06770" }, { "model": "GPT-4 Turbo + SWE-agent", "vendor": "openai", "date": "2024-05-06T00:00:00.000Z", "score": 12.47, "source": "independent", "source_url": "https://papers.neurips.cc/paper_files/paper/2024/file/5a7c947568c1b1328ccc5230172e1e7c-Paper-Conference.pdf" } ], "tags": [ "software-engineering", "github", "python", "repository-level", "unit-tests", "agentic" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Initial page. Timeline is restricted to the original 2,294-instance full-set protocol: paper Table 5 BM25 rows and SWE-agent paper Table 1. Verified and Lite scores are not mixed onto this axis." } ] }, { "name": "SWE-bench Pro", "slug": "swe-bench-pro", "capability": "coding", "secondary_capabilities": [ "agentic-tool-use", "long-context" ], "short_description": "Long-horizon repository tasks across public, held-out, and commercial codebases, with a continuing public model leaderboard.", "launch_date": "2025-09-19T00:00:00.000Z", "status": "active", "status_evidence": "The benchmark still has a maintained public leaderboard and substantial score spread: the 2026-08-03 LLM Stats snapshot ranges from 0.584 to 0.800 across these 17 leading models, despite documented task-quality concerns.", "saturated_date": null, "successor": null, "lineage": { "parent": "swe-bench", "variants": [ "swe-bench-pro-public", "swe-bench-pro-held-out", "swe-bench-pro-commercial" ] }, "paper": { "title": "SWE-Bench Pro: Can AI Agents Solve Long-Horizon Software Engineering Tasks?", "url": "https://arxiv.org/abs/2509.16941", "venue": null }, "authors": [ { "name": "Xiang Deng", "affiliation": "Scale AI" }, { "name": "Jeff Da", "affiliation": "Scale AI" }, { "name": "Edwin Pan", "affiliation": "Scale AI" }, { "name": "Yannis Yiming He", "affiliation": "Scale AI" }, { "name": "Brad Kenstler et al.", "affiliation": "Scale AI" } ], "maintainer": "Scale AI", "license": "MIT (benchmark code/metadata; source repositories retain their own licenses)", "dataset": { "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro", "size": 731, "public_test_set": true, "gated": false }, "metric": { "primary": "resolved instances on the public set (%)", "scoring": "unit-tests", "judge_model": null, "aggregation": "micro-average pass@1 over the 731-task public split under a common SWE-agent scaffold", "variants": [ { "name": "Commercial private split", "description": "Resolved percentage over 276 tasks from 18 proprietary startup repositories; results are reported but data are not public.", "scoring": "unit-tests", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "agentic", "temperature": null, "pass_at": 1, "tools": true, "harnesses": [ { "name": "Official SWE-bench Pro repository", "url": "https://github.com/scaleapi/SWE-bench_Pro-os", "task_flag": "swe_bench_pro_eval.py" } ] }, "cost_estimate": { "examples": 731, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No protocol-comparable aggregate human score is published. The authors describe tasks as requiring hours to days for professionals, and tasks have human reference solutions." }, "statistical_note": "At 731 items and 50% resolution, the nominal 95% binomial interval is about ±3.6 points. The 2026 audit's estimated ~30% broken-task rate dominates that sampling uncertainty. Dates on the 17 newest timeline points denote the 2026-08-03 LLM Stats snapshot, not individual evaluation or model-launch dates.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Public and held-out OSS repositories use strong-copyleft licenses as a legal deterrent to training inclusion; the commercial split uses private proprietary code. Neither is a technical guarantee.", "risk": "medium" }, "languages": [ "english", "python", "javascript", "typescript", "go" ], "leaderboards": [ { "name": "Official public leaderboard", "url": "https://labs.scale.com/leaderboard/swe_bench_pro_public", "type": "official" }, { "name": "Official commercial leaderboard", "url": "https://labs.scale.com/leaderboard/swe_bench_pro_private", "type": "official" }, { "name": "LLM Stats SWE-bench Pro leaderboard", "url": "https://llm-stats.com/benchmarks/swe-bench-pro", "type": "third-party" } ], "performance_timeline": [ { "model": "Qwen3 32B + SWE-agent (public, launch revision)", "vendor": "alibaba", "date": "2025-09-19T00:00:00.000Z", "score": 3.4, "source": "independent", "source_url": "https://static.scale.com/uploads/654197dc94d34f66c0f5184e/SWEAP_Eval_Scale%20%289%29.pdf" }, { "model": "GPT-4o + SWE-agent (public, launch revision)", "vendor": "openai", "date": "2025-09-19T00:00:00.000Z", "score": 4.9, "source": "independent", "source_url": "https://static.scale.com/uploads/654197dc94d34f66c0f5184e/SWEAP_Eval_Scale%20%289%29.pdf" }, { "model": "Claude Sonnet 4 + SWE-agent (public, launch revision)", "vendor": "anthropic", "date": "2025-09-19T00:00:00.000Z", "score": 17.6, "source": "independent", "source_url": "https://static.scale.com/uploads/654197dc94d34f66c0f5184e/SWEAP_Eval_Scale%20%289%29.pdf" }, { "model": "Claude Opus 4.1 + SWE-agent (public, launch revision)", "vendor": "anthropic", "date": "2025-09-19T00:00:00.000Z", "score": 22.7, "source": "independent", "source_url": "https://static.scale.com/uploads/654197dc94d34f66c0f5184e/SWEAP_Eval_Scale%20%289%29.pdf" }, { "model": "GPT-5 + SWE-agent (public, launch revision)", "vendor": "openai", "date": "2025-09-19T00:00:00.000Z", "score": 23.3, "source": "independent", "source_url": "https://static.scale.com/uploads/654197dc94d34f66c0f5184e/SWEAP_Eval_Scale%20%289%29.pdf" }, { "model": "Claude Fable 5 (LLM Stats snapshot)", "vendor": "anthropic", "date": "2026-08-03T00:00:00.000Z", "score": 80, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "Claude Mythos Preview (LLM Stats snapshot)", "vendor": "anthropic", "date": "2026-08-03T00:00:00.000Z", "score": 77.8, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "Claude Opus 4.8 (LLM Stats snapshot)", "vendor": "anthropic", "date": "2026-08-03T00:00:00.000Z", "score": 69.2, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "Grok 4.5 (LLM Stats snapshot)", "vendor": "xai", "date": "2026-08-03T00:00:00.000Z", "score": 64.7, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "GPT-5.6 Sol (LLM Stats snapshot)", "vendor": "openai", "date": "2026-08-03T00:00:00.000Z", "score": 64.6, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "Claude Opus 4.7 (LLM Stats snapshot)", "vendor": "anthropic", "date": "2026-08-03T00:00:00.000Z", "score": 64.3, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "GPT-5.6 Terra (LLM Stats snapshot)", "vendor": "openai", "date": "2026-08-03T00:00:00.000Z", "score": 63.4, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "Claude Sonnet 5 (LLM Stats snapshot)", "vendor": "anthropic", "date": "2026-08-03T00:00:00.000Z", "score": 63.2, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "GPT-5.6 Luna (LLM Stats snapshot)", "vendor": "openai", "date": "2026-08-03T00:00:00.000Z", "score": 62.7, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "GLM-5.2 (LLM Stats snapshot)", "vendor": "zhipu-ai", "date": "2026-08-03T00:00:00.000Z", "score": 62.1, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "Muse Spark 1.1 (LLM Stats snapshot)", "vendor": "meta", "date": "2026-08-03T00:00:00.000Z", "score": 61.5, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "Qwen3.7 Max (LLM Stats snapshot)", "vendor": "alibaba", "date": "2026-08-03T00:00:00.000Z", "score": 60.6, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "MiniMax M3 (LLM Stats snapshot)", "vendor": "minimax", "date": "2026-08-03T00:00:00.000Z", "score": 59, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "Gemini 3.6 Flash (LLM Stats snapshot)", "vendor": "google", "date": "2026-08-03T00:00:00.000Z", "score": 58.7, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "Kimi K2.6 (LLM Stats snapshot)", "vendor": "moonshot-ai", "date": "2026-08-03T00:00:00.000Z", "score": 58.6, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "GPT-5.5 (LLM Stats snapshot)", "vendor": "openai", "date": "2026-08-03T00:00:00.000Z", "score": 58.6, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" }, { "model": "GLM-5.1 (LLM Stats snapshot)", "vendor": "zhipu-ai", "date": "2026-08-03T00:00:00.000Z", "score": 58.4, "source": "vendor-reported", "source_url": "https://llm-stats.com/benchmarks/swe-bench-pro" } ], "tags": [ "software-engineering", "long-horizon", "private-code", "unit-tests", "agentic", "audit" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Reclassified as active and incorporated all 17 user-supplied rows from the 2026-08-03 LLM Stats snapshot, including the tied rank 15 and every supplied size, context, cost, and blank license value; retained the five launch-history points." }, { "date": "2026-08-03T00:00:00.000Z", "note": "Initial page. Marked deprecated after OpenAI's July 2026 audit. Timeline reproduces launch-paper Table 1 on the 731-task public split with one SWE-agent scaffold; later revised-dataset scores are excluded. Table 1 gives GPT-4o 4.9 although nearby prose says 3.9." } ] }, { "name": "SWE-bench Verified", "slug": "swe-bench-verified", "capability": "coding", "secondary_capabilities": [ "agentic-tool-use", "long-context" ], "short_description": "A 500-task, engineer-reviewed subset of SWE-bench that became the standard repository-level coding-agent test.", "launch_date": "2024-08-13T00:00:00.000Z", "status": "saturated", "status_evidence": "Classified saturated on 2026-02-17: under the standardized mini-SWE-agent 2.0.0 protocol, the leading results clustered at 76.8%, 75.8%, 75.8%, and 75.6%, limiting discrimination among frontier systems; separate contamination and task-quality concerns remain documented below.", "saturated_date": "2026-02-17T00:00:00.000Z", "successor": null, "lineage": { "parent": "swe-bench", "variants": [] }, "paper": { "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "url": "https://arxiv.org/abs/2310.06770", "venue": "ICLR 2024 Oral" }, "authors": [ { "name": "Neil Chowdhury et al.", "affiliation": "OpenAI" }, { "name": "Carlos E. Jimenez", "affiliation": "Princeton University" }, { "name": "John Yang", "affiliation": "Princeton University" } ], "maintainer": "SWE-bench team / OpenAI Preparedness", "license": "MIT (benchmark code; source repositories retain their own licenses)", "dataset": { "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified", "size": 500, "public_test_set": true, "gated": false }, "metric": { "primary": "resolved instances (%)", "scoring": "unit-tests", "judge_model": null, "aggregation": "micro-average over 500 verified instances; all FAIL_TO_PASS tests must pass and PASS_TO_PASS tests must remain passing", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "agentic", "temperature": null, "pass_at": 1, "tools": true, "harnesses": [ { "name": "Official SWE-bench harness", "url": "https://github.com/SWE-bench/SWE-bench", "task_flag": "SWE-bench_Verified" } ] }, "cost_estimate": { "examples": 500, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No protocol-comparable aggregate human pass rate was published. Annotators estimated fix time, and each task has a human-authored pull request, but neither is a blind human baseline." }, "statistical_note": "At 500 items, a 50% resolve rate has an approximate 95% binomial interval of ±4.4 points; repository clustering, flaky environments, and test defects add non-sampling uncertainty.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Three professional developers screened each candidate for issue and test quality, but the issues, gold patches, and repository histories remain public and static.", "risk": "high" }, "languages": [ "english", "python" ], "leaderboards": [ { "name": "Official SWE-bench Verified leaderboard", "url": "https://www.swebench.com", "type": "official" } ], "performance_timeline": [ { "model": "GPT-4o (Agentless)", "vendor": "openai", "date": "2024-08-13T00:00:00.000Z", "score": 33.2, "source": "vendor-reported", "source_url": "https://openai.com/index/introducing-swe-bench-verified/" }, { "model": "Claude 3.5 Sonnet (upgraded; Anthropic tools scaffold)", "vendor": "anthropic", "date": "2024-10-22T00:00:00.000Z", "score": 49, "source": "vendor-reported", "source_url": "https://www-cdn.anthropic.com/c6a80a657af445f40e31afac050f3bf76d3b1404.pdf" }, { "model": "Claude 4.5 Opus (high reasoning; mini-SWE-agent 2.0.0)", "vendor": "anthropic", "date": "2026-02-17T00:00:00.000Z", "score": 76.8, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "Gemini 3 Flash (high reasoning; mini-SWE-agent 2.0.0)", "vendor": "google", "date": "2026-02-17T00:00:00.000Z", "score": 75.8, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "MiniMax M2.5 (high reasoning; mini-SWE-agent 2.0.0)", "vendor": "minimax", "date": "2026-02-17T00:00:00.000Z", "score": 75.8, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "Claude Opus 4.6 (mini-SWE-agent 2.0.0)", "vendor": "anthropic", "date": "2026-02-17T00:00:00.000Z", "score": 75.6, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "GPT-5-2 Codex (mini-SWE-agent 2.0.0)", "vendor": "openai", "date": "2026-02-19T00:00:00.000Z", "score": 72.8, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "GLM-5 (high reasoning; mini-SWE-agent 2.0.0)", "vendor": "z-ai", "date": "2026-02-17T00:00:00.000Z", "score": 72.8, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "GPT-5-2 (high reasoning; mini-SWE-agent 2.0.0)", "vendor": "openai", "date": "2026-02-17T00:00:00.000Z", "score": 72.8, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "GPT 5.2 Codex (mini-SWE-agent 2.0.0)", "vendor": "openai", "date": "2026-02-19T00:00:00.000Z", "score": 72.8, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "Claude 4.5 Sonnet (high reasoning; mini-SWE-agent 2.0.0)", "vendor": "anthropic", "date": "2026-02-17T00:00:00.000Z", "score": 71.4, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "Kimi K2.5 (high reasoning; mini-SWE-agent 2.0.0)", "vendor": "moonshot-ai", "date": "2026-02-17T00:00:00.000Z", "score": 70.8, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "DeepSeek V3.2 (high reasoning; mini-SWE-agent 2.0.0)", "vendor": "deepseek", "date": "2026-02-17T00:00:00.000Z", "score": 70, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "Gemini 3 Pro (mini-SWE-agent 2.0.0)", "vendor": "google", "date": "2026-02-26T00:00:00.000Z", "score": 69.6, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "Claude 4.5 Haiku (high reasoning; mini-SWE-agent 2.0.0)", "vendor": "anthropic", "date": "2026-02-17T00:00:00.000Z", "score": 66.6, "source": "independent", "source_url": "https://www.swebench.com/" }, { "model": "GPT-5 Mini (mini-SWE-agent 2.0.0)", "vendor": "openai", "date": "2026-02-17T00:00:00.000Z", "score": 56.2, "source": "independent", "source_url": "https://www.swebench.com/" } ], "tags": [ "software-engineering", "github", "python", "verified", "unit-tests", "agentic", "contamination" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Reclassified as saturated and incorporated all 14 official mini-SWE-agent 2.0.0 results while preserving the two historical points, preserving ties, dates, costs, and both Codex name spellings." }, { "date": "2026-08-03T00:00:00.000Z", "note": "Initial page. Marked deprecated following OpenAI's 2026 audit. Timeline uses only official 500-task results and names each scaffold; OpenAI's internal 477-task system-card runs are excluded." } ] }, { "name": "SWE-Lancer", "slug": "swe-lancer", "capability": "coding", "secondary_capabilities": [ "agentic-tool-use", "reasoning" ], "short_description": "Current 198-task offline patch benchmark derived from paid Upwork work, with historical mixed implementation and manager-task variants.", "launch_date": "2025-02-17T00:00:00.000Z", "status": "deprecated", "status_evidence": "The maintained leaderboard is frozen at July 2025 and narrows paper-era Diamond from 502 mixed tasks to 198 offline IC tasks, making unlabeled Diamond claims non-comparable.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "swe-lancer-diamond-offline", "swe-lancer-diamond-original", "swe-lancer-full" ] }, "paper": { "title": "SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?", "url": "https://arxiv.org/abs/2502.12115", "venue": null }, "authors": [ { "name": "Samuel Miserendino", "affiliation": "OpenAI" }, { "name": "Michele Wang", "affiliation": "OpenAI" }, { "name": "Tejal Patwardhan", "affiliation": "OpenAI" }, { "name": "Johannes Heidecke", "affiliation": "OpenAI" } ], "maintainer": "OpenAI Frontier Evals", "license": "MIT (evaluation code)", "dataset": { "url": "https://github.com/openai/preparedness/tree/main/project/swelancer", "size": 198, "public_test_set": true, "gated": false }, "metric": { "primary": "pass@1 accuracy on the 198-task Diamond offline subset (%)", "scoring": "unit-tests", "judge_model": null, "aggregation": "micro-average over the current 198 offline IC software-engineering tasks; each submitted patch is graded by the task's end-to-end tests", "variants": [ { "name": "Earned payout", "description": "Sum of historical payouts attached to solved tasks on the current 198-task offline subset.", "scoring": "composite", "judge_model": null }, { "name": "Original Diamond overall accuracy", "description": "Historical paper protocol over 502 tasks: 237 executable IC tasks plus 265 manager proposal-choice tasks; not comparable to the current offline leaderboard.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "agentic", "temperature": null, "pass_at": 1, "tools": true, "harnesses": [ { "name": "OpenAI Frontier Evals", "url": "https://github.com/openai/preparedness", "task_flag": "project/swelancer" } ] }, "cost_estimate": { "examples": 198, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No blind protocol-comparable aggregate human baseline is reported. Original freelancers completed the IC work and hired managers chose proposals, defining the references rather than a timed test score." }, "statistical_note": "The current offline leaderboard has 198 executable patch tasks. The original paper's 502-task Diamond mixed 237 IC and 265 manager tasks, so its overall accuracy is a different axis.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "The benchmark uses a fixed historical codebase and, since July 2025, removes live Internet dependencies from execution; the public Diamond tasks and code are nevertheless exposed.", "risk": "medium" }, "languages": [ "english", "javascript", "typescript" ], "leaderboards": [ { "name": "Official SWE-Lancer leaderboard", "url": "https://swelancer.github.io/leaderboard/", "type": "official" } ], "performance_timeline": [ { "model": "GPT-4o (198-task Diamond offline)", "vendor": "openai", "date": "2025-07-17T00:00:00.000Z", "score": 8.1, "source": "vendor-reported", "source_url": "https://swelancer.github.io/leaderboard/" }, { "model": "o1 (198-task Diamond offline)", "vendor": "openai", "date": "2025-07-17T00:00:00.000Z", "score": 28.4, "source": "vendor-reported", "source_url": "https://swelancer.github.io/leaderboard/" } ], "tags": [ "software-engineering", "freelance", "economic-value", "upwork", "agentic", "unit-tests" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Initial page. Marked deprecated because the official leaderboard is frozen and overloads Diamond across incompatible 502-task and 198-task protocols. Timeline uses only the current 198-task offline leaderboard." } ] }, { "name": "TabMWP", "slug": "tabmwp", "capability": "maths", "secondary_capabilities": [ "reasoning", "multimodal" ], "short_description": "Grade-level math word problems that require combining a question with structured, textual, or image-rendered tables.", "launch_date": "2022-09-29T00:00:00.000Z", "status": "saturated", "status_evidence": "The official leaderboard reports Chameleon with GPT-4 at 98.78% on text-ground-truth tables, above the published 90.22% human estimate; the fixed public test set no longer separates frontier tool systems.", "saturated_date": "2023-04-19T00:00:00.000Z", "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "Dynamic Prompt Learning via Policy Gradient for Semi-structured Mathematical Reasoning", "url": "https://arxiv.org/abs/2209.14610", "venue": "ICLR 2023" }, "authors": [ { "name": "Pan Lu", "affiliation": "University of California, Los Angeles" }, { "name": "Liang Qiu", "affiliation": null }, { "name": "Kai-Wei Chang", "affiliation": "University of California, Los Angeles" }, { "name": "Peter Clark", "affiliation": "Allen Institute for AI" } ], "maintainer": "PromptPG authors", "license": "MIT", "dataset": { "url": "https://github.com/lupantech/PromptPG/tree/main/data/tabmwp", "size": 7686, "public_test_set": true, "gated": false }, "metric": { "primary": "answer accuracy (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "mean accuracy on the 7,686-item test split; official breakdowns separate free-text/multiple-choice and answer types", "variants": [ { "name": "Image-table input", "description": "The table is rendered as an image, adding table perception to mathematical reasoning.", "scoring": "exact-match", "judge_model": null }, { "name": "Text-GT input", "description": "The gold semi-structured table text is supplied, isolating reasoning from OCR and table recognition.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "2-shot for the original GPT-3 PromptPG result; leaderboard protocols vary", "cot": "cot", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "PromptPG reference implementation", "url": "https://github.com/lupantech/PromptPG", "task_flag": "data/tabmwp" } ] }, "cost_estimate": { "examples": 7686, "approx_usd_frontier_model": null }, "human_baseline": { "score": 90.22, "note": "Official test-set estimate reported by the benchmark authors; subgroup accuracy varies by free-text, multiple-choice, and answer type." }, "statistical_note": "The test split is large, but errors are clustered by source table and answer format; close results must use the same table representation, prompt selection, tools, and parsing rules.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "None beyond fixed train/dev/test splits; all tables, solutions, and test answers are public in the repository.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official PromptPG leaderboard", "url": "https://github.com/lupantech/PromptPG", "type": "official" } ], "performance_timeline": [ { "model": "GPT-3 + PromptPG (2-shot CoT, Text-GT)", "vendor": "openai", "date": "2022-09-29T00:00:00.000Z", "score": 68.23, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2209.14610" }, { "model": "Chameleon (GPT-4, Text-GT, tools)", "vendor": "openai", "date": "2023-04-19T00:00:00.000Z", "score": 98.78, "source": "independent", "source_url": "https://arxiv.org/abs/2304.09842" } ], "tags": [ "maths", "tables", "multimodal", "word-problems", "question-answering" ], "last_updated": "2026-07-29T00:00:00.000Z", "changelog": [ { "date": "2026-07-29T00:00:00.000Z", "note": "Initial page. Timeline is restricted to official Text-GT test accuracy; image-input and differently tooled protocols are not mixed into that axis." } ] }, { "name": "Terminal-Bench", "slug": "terminal-bench", "capability": "coding", "secondary_capabilities": [ "agentic-tool-use" ], "short_description": "Eighty-nine realistic terminal tasks graded in containers; version 2.1 repairs 2.0's dependency, resource, and specification defects.", "launch_date": "2025-11-07T00:00:00.000Z", "status": "active", "status_evidence": "Terminal-Bench 2.1 remains actively maintained and submission-driven; its verified 17-entry leaderboard spans 58.7% to 83.8% across multiple agent-model pairs, while Terminal-Bench 3.0 is still in development.", "saturated_date": null, "successor": null, "lineage": { "parent": "terminal-bench-1", "variants": [ "terminal-bench-2-0", "terminal-bench-2-1" ] }, "paper": { "title": "Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces", "url": "https://arxiv.org/abs/2601.11868", "venue": "ICLR 2026" }, "authors": [ { "name": "Mike A. Merrill", "affiliation": "Stanford University" }, { "name": "Alexander G. Shaw", "affiliation": "Laude Institute" }, { "name": "Nicholas Carlini", "affiliation": "Anthropic" }, { "name": "Boxuan Li et al.", "affiliation": null } ], "maintainer": "Terminal-Bench team / Harbor Framework", "license": "Apache-2.0", "dataset": { "url": "https://github.com/harbor-framework/terminal-bench-2-1", "size": 89, "public_test_set": true, "gated": false }, "metric": { "primary": "task accuracy on Terminal-Bench 2.1 (%)", "scoring": "unit-tests", "judge_model": null, "aggregation": "mean task resolution over 89 tasks; official leaderboard submissions require at least five public trials per task", "variants": [ { "name": "Terminal-Bench 2.0 accuracy", "description": "Resolution over the predecessor's 89-task definitions; not directly comparable because 2.1 changes 28 tasks and their environments or resources.", "scoring": "unit-tests", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "agentic", "temperature": null, "pass_at": 1, "tools": true, "harnesses": [ { "name": "Harbor", "url": "https://github.com/harbor-framework/harbor", "task_flag": "terminal-bench/terminal-bench-2-1" } ] }, "cost_estimate": { "examples": 89, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No aggregate blind human baseline is published. Every 2.0 task has a human-written solution and expert-time metadata, establishing solvability but not a comparable score." }, "statistical_note": "With only 89 tasks, one task changes accuracy by about 1.12 points and a 50% pass rate has a nominal 95% interval near ±10.4 points. Agent/model pair, trial count, resources, and task version are essential.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Task repositories include a canary asking training pipelines to exclude benchmark data; 2.1 adds continuous dependency and oracle validation, but neither prevents public-task memorization.", "risk": "high" }, "languages": [ "english", "shell", "python", "c", "cpp", "rust", "javascript" ], "leaderboards": [ { "name": "Official Terminal-Bench 2.1 leaderboard", "url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1", "type": "official" } ], "performance_timeline": [ { "model": "Claude Code + Fable 5 (xhigh; TB 2.1)", "vendor": "anthropic", "date": "2026-06-07T00:00:00.000Z", "score": 83.8, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Codex + GPT-5.5 (xhigh; TB 2.1)", "vendor": "openai", "date": "2026-05-01T00:00:00.000Z", "score": 83.1, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Terminus 2 + Fable 5 (high; TB 2.1)", "vendor": "anthropic", "date": "2026-06-05T00:00:00.000Z", "score": 80.4, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Cursor CLI + Grok 4.5 (high; TB 2.1)", "vendor": "xai", "date": "2026-07-09T00:00:00.000Z", "score": 79.3, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Claude Code + Opus 4.8 (high; TB 2.1)", "vendor": "anthropic", "date": "2026-07-09T00:00:00.000Z", "score": 78.9, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Codex + GPT-5.6 Terra (max; TB 2.1)", "vendor": "openai", "date": "2026-07-11T00:00:00.000Z", "score": 78.4, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Terminus 2 + GPT-5.5 (xhigh; TB 2.1)", "vendor": "openai", "date": "2026-05-01T00:00:00.000Z", "score": 78, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "mini-SWE-agent + Muse Spark 1.1 (xhigh; TB 2.1)", "vendor": "meta", "date": "2026-07-09T00:00:00.000Z", "score": 76.2, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Codex + GPT-5.6 Luna (max; TB 2.1)", "vendor": "openai", "date": "2026-07-11T00:00:00.000Z", "score": 75.7, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Claude Code + Sonnet 5 (high; TB 2.1)", "vendor": "anthropic", "date": "2026-07-09T00:00:00.000Z", "score": 74.6, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Terminus 2 + Gemini 3 Pro (high; TB 2.1)", "vendor": "google", "date": "2026-05-01T00:00:00.000Z", "score": 73.9, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Claude Code + Opus 4.7 (max; TB 2.1)", "vendor": "anthropic", "date": "2026-05-01T00:00:00.000Z", "score": 68.9, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Terminus 2 + Opus 4.7 (max; TB 2.1)", "vendor": "anthropic", "date": "2026-05-01T00:00:00.000Z", "score": 66.1, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Gemini CLI + Gemini 3 Pro (high; TB 2.1)", "vendor": "google", "date": "2026-05-01T00:00:00.000Z", "score": 65.8, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Gemini CLI + Gemini 3.1 Pro (high; TB 2.1)", "vendor": "google", "date": "2026-05-05T00:00:00.000Z", "score": 65.8, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Terminus 2 + Gemini 3.1 Pro (high; TB 2.1)", "vendor": "google", "date": "2026-05-05T00:00:00.000Z", "score": 65.6, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" }, { "model": "Claude Code + GLM-5.1 (max; TB 2.1)", "vendor": "z-ai", "date": "2026-05-01T00:00:00.000Z", "score": 58.7, "source": "independent", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1" } ], "tags": [ "terminal", "agentic", "containers", "software-engineering", "system-administration", "security" ], "last_updated": "2026-08-03T00:00:00.000Z", "changelog": [ { "date": "2026-08-03T00:00:00.000Z", "note": "Expanded the Terminal-Bench 2.1 timeline from four to all 17 verified leaderboard entries and added every supplied rank, agent, model, effort, confidence interval, date, organization, PR, hack-rate, and cost field." }, { "date": "2026-08-03T00:00:00.000Z", "note": "Initial page. Structured fields and timeline use only verified Terminal-Bench 2.1 leaderboard rows with the exact agent, effort, and five-trial submission protocol; 2.0 scores are not mixed into the chart." } ] }, { "name": "Tool Decathlon / Toolathlon", "slug": "toolathlon", "capability": "agentic-tool-use", "secondary_capabilities": [ "instruction-following", "long-context", "coding" ], "short_description": "A long-horizon tool-use benchmark with 108 tasks across 32 software applications, 604 MCP tools, and state-based evaluators.", "launch_date": "2025-10-29T00:00:00.000Z", "status": "active", "status_evidence": "Toolathlon-Verified is the current benchmark release as of 2026-06-30. The official site reports 108 tasks, 32 MCP servers with 604 tools, 7 local toolkits, and a current Verified leaderboard.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "toolathlon-verified", "toolathlon-sandbox" ] }, "paper": { "title": "The Tool Decathlon: Benchmarking Language Agents for Diverse, Realistic, and Long-Horizon Task Execution", "url": "https://arxiv.org/abs/2510.25726", "venue": "ICLR 2026" }, "authors": [ { "name": "Junlong Li", "affiliation": "HKUST" }, { "name": "Wenshuo Zhao", "affiliation": null }, { "name": "Jian Zhao", "affiliation": null }, { "name": "Weihao Zeng", "affiliation": null }, { "name": "Haoze Wu et al.", "affiliation": null } ], "maintainer": "HKUST-NLP Toolathlon team", "license": null, "dataset": { "url": "https://github.com/hkust-nlp/Toolathlon", "size": 108, "public_test_set": true, "gated": false }, "metric": { "primary": "Pass@1 on Toolathlon-Verified (%)", "scoring": "composite", "judge_model": null, "aggregation": "task-level pass/fail from dedicated state-based evaluation scripts, aggregated over 108 Toolathlon-Verified tasks; Pass@3 and Pass^3 are reported separately", "variants": [ { "name": "Pass@3", "description": "Whether at least one of three attempts succeeds, reported by the official leaderboard for supported runs.", "scoring": "composite", "judge_model": null }, { "name": "Pass^3", "description": "A stricter three-run consistency view reported by the official leaderboard for supported runs.", "scoring": "composite", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "agentic", "temperature": null, "pass_at": 1, "tools": true, "harnesses": [ { "name": "Official Toolathlon harness", "url": "https://github.com/hkust-nlp/Toolathlon", "task_flag": "finalpool/{task_name}" }, { "name": "Toolathlon public evaluation service", "url": "https://toolathlon.xyz/docs/test", "task_flag": "public" } ] }, "cost_estimate": { "examples": 108, "approx_usd_frontier_model": "large and model-dependent; official results report average turns and tool calls because long-horizon workflows can require roughly 20 turns per task" }, "human_baseline": { "score": null, "note": "No single protocol-comparable human baseline is published for the full Toolathlon-Verified set." }, "statistical_note": "Toolathlon-Verified starts a new official score series. Do not compare Verified results directly with the archived pre-Verified leaderboard.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Toolathlon-Verified reviewed prompts, ground truths, evaluators, initial states, and infrastructure; state-based scripts improve grading reliability but do not hide public tasks.", "risk": "high" }, "languages": [ "english", "shell", "python", "sql" ], "leaderboards": [ { "name": "Toolathlon model leaderboard", "url": "https://toolathlon.xyz/docs/leaderboard", "type": "official" } ], "performance_timeline": [ { "model": "Claude Opus 4.8 (max; Toolathlon-Verified)", "vendor": "anthropic", "date": "2026-05-28T00:00:00.000Z", "score": 76.2, "source": "independent", "source_url": "https://toolathlon.xyz/docs/leaderboard" }, { "model": "Kimi K3 (max; Toolathlon-Verified)", "vendor": "moonshot", "date": "2026-07-16T00:00:00.000Z", "score": 76.5, "source": "independent", "source_url": "https://toolathlon.xyz/docs/leaderboard" }, { "model": "Muse Spark 1.2 (xhigh; Toolathlon-Verified)", "vendor": "meta", "date": "2026-08-05T00:00:00.000Z", "score": 75.9, "source": "independent", "source_url": "https://toolathlon.xyz/docs/leaderboard" } ], "tags": [ "agents", "mcp", "tool-use", "long-horizon", "workflows", "applications", "state-based-evaluation" ], "last_updated": "2026-08-30T00:00:00.000Z", "changelog": [ { "date": "2026-08-30T00:00:00.000Z", "note": "Reworked graph points to use model launch dates and removed the pre-release Claude-4.5-Sonnet paper point from the timeline." }, { "date": "2026-08-30T00:00:00.000Z", "note": "Initial page. Separates the arXiv pre-Verified baseline from the Toolathlon-Verified leaderboard because the official site says the score series is not directly comparable." } ] }, { "name": "TruthfulQA", "slug": "truthfulqa", "capability": "safety-alignment", "secondary_capabilities": [], "short_description": "817 adversarial questions probing whether models repeat common human misconceptions — famous for finding that bigger models were often less truthful.", "launch_date": "2021-09-08T00:00:00.000Z", "status": "saturated", "status_evidence": "By 2023 post-RLHF models exceeded the 94% human truthfulness baseline — but largely because the exact misconceptions TruthfulQA targets are what alignment tuning trains against, and the dataset itself leaked into alignment mixes. Saturation here reflects targeted contamination, not broadly truthful models. saturated_date (2023-06) marks the ChatGPT/GPT-4 era when the inverse-scaling trend reversed.", "saturated_date": "2023-06-01T00:00:00.000Z", "successor": null, "lineage": { "parent": null, "variants": [] }, "paper": { "title": "TruthfulQA: Measuring How Models Mimic Human Falsehoods", "url": "https://arxiv.org/abs/2109.07958", "venue": "ACL 2022" }, "authors": [ { "name": "Stephanie Lin", "affiliation": "University of Oxford" }, { "name": "Jacob Hilton", "affiliation": "OpenAI" }, { "name": "Owain Evans", "affiliation": "University of Oxford" } ], "maintainer": "Stephanie Lin / Owain Evans (github.com/sylinrl/TruthfulQA)", "license": "Apache-2.0", "dataset": { "url": "https://huggingface.co/datasets/domenicrosati/TruthfulQA", "size": 817, "public_test_set": true, "gated": false }, "metric": { "primary": "MC2 (normalized probability mass on true answers, %)", "scoring": "log-likelihood", "judge_model": null, "aggregation": "mean over 817 questions (no category weighting)", "variants": [ { "name": "MC1", "description": "Single-true multiple choice: accuracy at assigning the highest probability to the one correct answer among 4–5 options.", "scoring": "exact-match", "judge_model": null }, { "name": "MC2", "description": "Multi-true multiple choice: the normalized total probability the model places on the set of true answers. The most-reported number, and the site's headline metric.", "scoring": "log-likelihood", "judge_model": null }, { "name": "generation (truthful %)", "description": "Open-ended answer graded for truthfulness (and separately informativeness). The task with a human baseline, and the axis the timeline uses.", "scoring": "llm-judge", "judge_model": "GPT-judge — a fine-tuned GPT-3 curie, now deprecated; commonly replaced by GPT-4-as-judge, which makes cross-year generation scores non-comparable." } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "task_flag": "truthfulqa_mc2" }, { "name": "HELM", "url": "https://crfm.stanford.edu/helm", "task_flag": "truthful_qa" } ] }, "cost_estimate": { "examples": 817, "approx_usd_frontier_model": "1–5 (817 short items; MC1/MC2 need only log-probs. The generation task adds a GPT-4-judge pass, which costs more)" }, "human_baseline": { "score": 94, "note": "94% truthful on the generation task in the original paper (human validators). No published human baseline exists for the MC1/MC2 multiple-choice variants, so the timeline and gauge track generation truthfulness to stay comparable to this number." }, "statistical_note": "817 questions → roughly ±3pt across the 60–90% range (95% CI). The 38 category splits (some <20 questions) are far too small for per-category comparison. Metric choice (MC1 vs MC2 vs generation) moves scores more than most model-to-model gaps do.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "None, and uniquely self-defeating: the misconceptions TruthfulQA targets are exactly what RLHF and Constitutional-style alignment tuning is trained to avoid, and the dataset itself appears in some alignment/instruction mixes. Modern scores partly measure exposure to the benchmark's own patterns.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Open LLM Leaderboard (v1, archived)", "url": "https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard", "type": "third-party" } ], "performance_timeline": [ { "model": "GPT-3 175B", "vendor": "openai", "date": "2021-09-08T00:00:00.000Z", "score": 58, "source": "independent", "source_url": "https://arxiv.org/abs/2109.07958" } ], "tags": [ "truthfulness", "hallucination", "alignment", "adversarial", "multiple-choice" ], "last_updated": "2026-07-22T00:00:00.000Z", "changelog": [ { "date": "2026-07-22T00:00:00.000Z", "note": "Updated dataset URL to the domenicrosati/TruthfulQA HuggingFace mirror." }, { "date": "2026-07-21T00:00:00.000Z", "note": "Initial page. Timeline is deliberately a single sourced point (GPT-3 58% truthful, original paper) on the generation-truthfulness axis — the only axis with a human baseline. Later comparable points are omitted, not fabricated: the GPT-judge deprecation makes cross-year generation scores non-comparable, and modern MC2 numbers are harness-dependent with no authoritative single source (both documented in Known issues). Schema extended this commit: added `log-likelihood` to the scoring enum and an optional `metric.variants[]` array to model MC1/MC2/generation honestly." } ] }, { "name": "Video-MME", "slug": "video-mme", "capability": "multimodal", "secondary_capabilities": [ "long-context", "reasoning" ], "short_description": "Video-understanding benchmark with 900 videos, 2,700 questions, three duration bands, subtitles, and audio.", "launch_date": "2024-05-31T00:00:00.000Z", "status": "deprecated", "status_evidence": "The maintainers launched Video-MME-v2 in 2026 because existing video benchmarks were saturating and did not match user experience; new evaluations should prefer v2's grouped temporal-reasoning protocol.", "saturated_date": null, "successor": "video-mme-v2", "lineage": { "parent": null, "variants": [ "video-mme-v2" ] }, "paper": { "title": "Video-MME: The First-Ever Comprehensive Evaluation Benchmark of Multi-modal LLMs in Video Analysis", "url": "https://arxiv.org/abs/2405.21075v3", "venue": "CVPR 2025" }, "authors": [ { "name": "Chaoyou Fu", "affiliation": "Nanjing University" }, { "name": "Yuhan Dai", "affiliation": "University of Science and Technology of China" }, { "name": "Yongdong Luo et al.", "affiliation": null } ], "maintainer": "MME Benchmarks / Video-MME team", "license": null, "dataset": { "url": "https://huggingface.co/datasets/lmms-lab/Video-MME", "size": 2700, "public_test_set": true, "gated": false }, "metric": { "primary": "multiple-choice accuracy (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "micro accuracy across 2,700 four-option questions; also report short, medium, and long duration bands and whether subtitles/audio were supplied", "variants": [ { "name": "without subtitles", "description": "Video frames only; frame count and sampling policy must be reported.", "scoring": "exact-match", "judge_model": null }, { "name": "with subtitles", "description": "Adds subtitle text where supported — the leaderboard's default sort and the headline condition for this page. Subtitle and audio gains are reported separately from frames-only results.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official Video-MME code", "url": "https://github.com/MME-Benchmarks/Video-MME", "task_flag": "Video-MME" } ] }, "cost_estimate": { "examples": 2700, "approx_usd_frontier_model": "high and highly dependent on sampled frames, video resolution, subtitles, and native-audio support" }, "human_baseline": { "score": null, "note": "No single protocol-comparable human aggregate for the original Video-MME is recorded here." }, "statistical_note": "Frame sampling can move scores materially. Comparisons must match the number/resolution of frames and the subtitle/audio condition; duration-band averages should accompany the overall score.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Questions were newly human-authored and filtered against text-only answering, but the videos are public YouTube material and the complete annotations are now public.", "risk": "high" }, "languages": [ "english", "chinese", "japanese", "other-multilingual-video" ], "leaderboards": [ { "name": "Official Video-MME leaderboard", "url": "https://video-mme.github.io/home_page.html#leaderboard", "type": "official" } ], "performance_timeline": [ { "model": "GPT-4o (with subtitles, 384 frames)", "vendor": "openai", "date": "2024-06-15T00:00:00.000Z", "score": 77.2, "source": "independent", "source_url": "https://video-mme.github.io/home_page.html#leaderboard" }, { "model": "Gemini 1.5 Pro (with subtitles, 1/0.5 fps)", "vendor": "google", "date": "2024-06-15T00:00:00.000Z", "score": 81.3, "source": "independent", "source_url": "https://video-mme.github.io/home_page.html#leaderboard" } ], "tags": [ "video", "temporal-reasoning", "long-video", "subtitles", "audio", "mcq" ], "last_updated": "2026-09-05T00:00:00.000Z", "changelog": [ { "date": "2026-09-05T00:00:00.000Z", "note": "Switch headline performance_timeline to with-subtitles scores (GPT-4o 77.2, Gemini 1.5 Pro 81.3) per official leaderboard default sort; update leaderboard link to home_page.html#leaderboard." }, { "date": "2026-09-05T00:00:00.000Z", "note": "Initial page based on arXiv v3 and official repositories; marked deprecated in favor of the maintainers' Video-MME-v2." } ] }, { "name": "WildGuardTest", "slug": "wildguardtest", "capability": "safety-alignment", "secondary_capabilities": [ "instruction-following" ], "short_description": "A human-annotated test set for prompt harmfulness, response harmfulness, and refusal classification.", "launch_date": "2024-06-26T00:00:00.000Z", "status": "active", "status_evidence": "WildGuardTest remains a widely used open moderation test with adversarial and in-the-wild examples across three safety-classification tasks, though its public English-only set is not a deployment policy.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "wildguardmix", "wildguardtrain", "xstest-response" ] }, "paper": { "title": "WildGuard: Open One-Stop Moderation Tools for Safety Risks, Jailbreaks, and Refusals of LLMs", "url": "https://arxiv.org/html/2406.18495v3", "venue": "NeurIPS 2024 Datasets and Benchmarks Track" }, "authors": [ { "name": "Seungju Han", "affiliation": "Seoul National University" }, { "name": "Kavel Rao", "affiliation": null }, { "name": "Allyson Ettinger", "affiliation": "Allen Institute for AI" }, { "name": "Liwei Jiang", "affiliation": "University of Washington / Allen Institute for AI" }, { "name": "Bill Yuchen Lin", "affiliation": "Allen Institute for AI" }, { "name": "Nathan Lambert", "affiliation": "Allen Institute for AI" }, { "name": "Yejin Choi", "affiliation": "University of Washington / Allen Institute for AI" }, { "name": "Nouha Dziri", "affiliation": "Allen Institute for AI" } ], "maintainer": "Allen Institute for AI", "license": "ODC-By data; Apache-2.0 WildGuard model", "dataset": { "url": "https://huggingface.co/datasets/allenai/wildguardmix", "size": 1725, "public_test_set": true, "gated": true }, "metric": { "primary": "prompt-harmfulness total F1 (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "timeline plots total F1 for prompt harmfulness; response-harmfulness and refusal F1 remain separate companion metrics, and vanilla and adversarial subsets should also remain visible", "variants": [ { "name": "prompt harmfulness", "description": "Binary harmful versus unharmful classification of the user prompt.", "scoring": "exact-match", "judge_model": null }, { "name": "response harmfulness", "description": "Binary harmful versus unharmful classification of the assistant response.", "scoring": "exact-match", "judge_model": null }, { "name": "response refusal", "description": "Binary refusal versus compliance classification for prompt-response pairs.", "scoring": "exact-match", "judge_model": null } ] }, "run_config": { "shots": "zero-shot classifier inference", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Ai2 Safety Tool", "url": "https://github.com/allenai/safety-eval", "task_flag": "wildguardtest_prompt,wildguardtest_response,wildguardtest_refusal" } ] }, "cost_estimate": { "examples": 1725, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "The labels come from three independent annotators per prompt-response pair, but no single human model-style F1 baseline is reported." }, "statistical_note": "The released test split has 1,725 prompt-response items. Counting three label tasks yields roughly 5K labeled decisions, which should not be reported as 5,000 independent conversations. The timeline plots prompt-harmfulness total F1 only. Timeline dates are moderator-model public-launch dates, not evaluation dates.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Access is gated by Ai2 responsible-use terms. Keep the official split untouched, report task and adversarial subset, and evaluate on private policy-specific traffic before deployment.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [], "performance_timeline": [ { "model": "GPT-4 judge (prompt-harmfulness total F1)", "vendor": "openai", "date": "2023-03-14T00:00:00.000Z", "score": 87.9, "source": "independent", "source_url": "https://arxiv.org/html/2406.18495v3" }, { "model": "Llama Guard 2 (prompt-harmfulness total F1)", "vendor": "meta", "date": "2024-04-18T00:00:00.000Z", "score": 70.9, "source": "independent", "source_url": "https://arxiv.org/html/2406.18495v3" }, { "model": "WildGuard (prompt-harmfulness total F1)", "vendor": "ai2", "date": "2024-06-26T00:00:00.000Z", "score": 88.9, "source": "independent", "source_url": "https://arxiv.org/html/2406.18495v3" } ], "tags": [ "moderation", "harmfulness", "refusal", "jailbreaks", "classifier-evaluation" ], "last_updated": "2026-09-14T00:00:00.000Z", "changelog": [ { "date": "2026-09-14T00:00:00.000Z", "note": "Added paper-reported prompt-harmfulness F1 using verified moderator-model public-launch dates for the timeline x-axis." }, { "date": "2026-09-14T00:00:00.000Z", "note": "Initial page based on arXiv v3, the WildGuardMix dataset card, WildGuard package, and Ai2 Safety Tool." } ] }, { "name": "WinoGrande", "slug": "winogrande", "capability": "reasoning", "secondary_capabilities": [ "knowledge" ], "short_description": "Binary fill-in-the-blank commonsense problems, scaled from Winograd schemas and adversarially filtered to reduce dataset shortcuts.", "launch_date": "2019-07-24T00:00:00.000Z", "status": "saturated", "status_evidence": "The original XL result was 79.1% against a 94.0% human estimate, leaving real headroom, but the static public inputs and widespread validation-set use make modern scores contamination-sensitive.", "saturated_date": "2022-02-17T00:00:00.000Z", "successor": null, "lineage": { "parent": "winograd-schema-challenge", "variants": [ "winogrande-debiased", "winogrande-xs", "winogrande-s", "winogrande-m", "winogrande-l", "winogrande-xl" ] }, "paper": { "title": "WinoGrande: An Adversarial Winograd Schema Challenge at Scale", "url": "https://arxiv.org/abs/1907.10641", "venue": "AAAI 2020" }, "authors": [ { "name": "Keisuke Sakaguchi", "affiliation": null }, { "name": "Ronan Le Bras", "affiliation": null }, { "name": "Chandra Bhagavatula", "affiliation": null }, { "name": "Yejin Choi", "affiliation": null } ], "maintainer": "Allen Institute for AI", "license": "CC BY (dataset); Apache-2.0 (code)", "dataset": { "url": "https://huggingface.co/datasets/allenai/winogrande", "size": 1767, "public_test_set": false, "gated": false }, "metric": { "primary": "accuracy (%)", "scoring": "exact-match", "judge_model": null, "aggregation": "Mean correctness over examples; each prediction is option 1 or option 2, so random chance is 50%. Official test labels are hidden and evaluated by submission.", "variants": [ { "name": "lm-eval likelihood-ranked accuracy", "description": "Ranks the two candidate substitutions by conditional log-likelihood on the public 1,267-example validation split; this is not the hidden-label official test protocol.", "scoring": "log-likelihood", "judge_model": null } ] }, "run_config": { "shots": "zero-shot by default in lm-evaluation-harness; original paper fine-tunes on XS/S/M/L/XL training subsets", "cot": "direct", "temperature": null, "pass_at": 1, "tools": false, "harnesses": [ { "name": "WinoGrande official code", "url": "https://github.com/allenai/winogrande", "task_flag": "winogrande" }, { "name": "lm-evaluation-harness", "url": "https://github.com/EleutherAI/lm-evaluation-harness/tree/main/lm_eval/tasks/winogrande", "task_flag": "winogrande" } ] }, "cost_estimate": { "examples": 1267, "approx_usd_frontier_model": null }, "human_baseline": { "score": 94, "note": "Crowd-worker estimate reported by the original paper; compare with care because current lm-eval runs use the validation split rather than the hidden-label official test set." }, "statistical_note": "On the 1,267-example validation split, a 50% accuracy estimate has an approximate 95% binomial margin of error of +/-2.8 points; paired predictions are preferable for close model comparisons.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Official test labels remain hidden, and lm-evaluation-harness exposes a decontamination query, but the test inputs and the fully labeled validation split are public and unchanged.", "risk": "high" }, "languages": [ "english" ], "leaderboards": [ { "name": "Official WinoGrande leaderboard", "url": "https://leaderboard.allenai.org/winogrande/submissions/public", "type": "official" } ], "performance_timeline": [ { "model": "BERT-Large (fine-tuned)", "vendor": "google", "date": "2019-07-24T00:00:00.000Z", "score": 64.9, "source": "independent", "source_url": "https://arxiv.org/abs/1907.10641" }, { "model": "RoBERTa-large (fine-tuned)", "vendor": "facebook-ai", "date": "2019-07-24T00:00:00.000Z", "score": 79.1, "source": "independent", "source_url": "https://arxiv.org/abs/1907.10641" }, { "model": "GPT-3 175B (few-shot)", "vendor": "openai", "date": "2020-07-22T00:00:00.000Z", "score": 77.7, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2005.14165" }, { "model": "PaLM 540B (5-shot)", "vendor": "google", "date": "2022-04-04T00:00:00.000Z", "score": 85.1, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2204.02311" }, { "model": "GPT-4 (5-shot)", "vendor": "openai", "date": "2023-03-14T00:00:00.000Z", "score": 87.5, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2303.08774" }, { "model": "Llama 2 70B (5-shot)", "vendor": "meta", "date": "2023-07-18T00:00:00.000Z", "score": 81.8, "source": "vendor-reported", "source_url": "https://arxiv.org/abs/2307.09288" }, { "model": "Claude 3 Opus (5-shot)", "vendor": "anthropic", "date": "2024-03-04T00:00:00.000Z", "score": 88.5, "source": "vendor-reported", "source_url": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf" }, { "model": "Llama 3 70B (5-shot)", "vendor": "meta", "date": "2024-04-18T00:00:00.000Z", "score": 83.1, "source": "vendor-reported", "source_url": "https://ai.meta.com/blog/meta-llama-3/" }, { "model": "Llama 3.1 405B (5-shot)", "vendor": "meta", "date": "2024-07-23T00:00:00.000Z", "score": 89.2, "source": "vendor-reported", "source_url": "https://ai.meta.com/blog/meta-llama-3-1/" } ], "tags": [ "commonsense", "coreference", "pronoun-resolution", "winograd", "adversarial-filtering", "binary-choice" ], "last_updated": "2026-07-25T00:00:00.000Z", "changelog": [ { "date": "2026-07-25T00:00:00.000Z", "note": "Added performance timeline with fine-tuned and few-shot scores from BERT-Large, RoBERTa, GPT-3, PaLM, GPT-4, Llama 2/3, and Claude 3 Opus. Model names include protocol notes (fine-tuned vs few-shot) because protocol differences dominate comparability." } ] }, { "name": "XSTest", "slug": "xstest", "capability": "safety-alignment", "secondary_capabilities": [ "instruction-following" ], "short_description": "A 450-prompt test of whether models refuse unsafe requests without over-refusing superficially similar safe ones.", "launch_date": "2023-08-02T00:00:00.000Z", "status": "active", "status_evidence": "XSTest remains a useful two-sided calibration diagnostic because it measures exaggerated refusal alongside harmful compliance, though its small public set should be paired with broader tests.", "saturated_date": null, "successor": null, "lineage": { "parent": null, "variants": [ "xstest-safe", "xstest-unsafe-contrast", "xstest-response" ] }, "paper": { "title": "XSTest: A Test Suite for Identifying Exaggerated Safety Behaviours in Large Language Models", "url": "https://arxiv.org/html/2308.01263v3", "venue": "NAACL 2024" }, "authors": [ { "name": "Paul Röttger", "affiliation": "Bocconi University" }, { "name": "Hannah Rose Kirk", "affiliation": "University of Oxford" }, { "name": "Bertie Vidgen", "affiliation": "University of Oxford" }, { "name": "Giuseppe Attanasio", "affiliation": "Bocconi University" }, { "name": "Federico Bianchi", "affiliation": "Stanford University" }, { "name": "Dirk Hovy", "affiliation": "Bocconi University" } ], "maintainer": "XSTest authors", "license": "CC-BY-4.0 prompts; model completions retain provider licenses", "dataset": { "url": "https://github.com/paul-rottger/xstest", "size": 450, "public_test_set": true, "gated": false }, "metric": { "primary": "unsafe-prompt total refusal rate (%)", "scoring": "composite", "judge_model": null, "aggregation": "manual full-refusal plus partial-refusal percentage over 200 unsafe contrast prompts; always report the companion safe-prompt refusal rate to expose over-refusal", "variants": [ { "name": "safe prompt refusal rate", "description": "Fraction of benign prompts refused; lower is better and directly measures exaggerated safety.", "scoring": "human-graded", "judge_model": null }, { "name": "unsafe prompt refusal rate", "description": "Fraction of contrast prompts refused; higher is safer for the intended general-assistant setting.", "scoring": "human-graded", "judge_model": null } ] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Official XSTest evaluation", "url": "https://github.com/paul-rottger/xstest", "task_flag": "evaluation/classify_completions.py" } ] }, "cost_estimate": { "examples": 450, "approx_usd_frontier_model": "low for generation; manual response labeling or an external classifier adds cost" }, "human_baseline": { "score": null, "note": "No single human score is appropriate. Humans created and labeled the prompts and response taxonomy; the desired direction is compliance on safe prompts and refusal on unsafe prompts." }, "statistical_note": "With 250 safe and 200 unsafe prompts, a few responses move category rates visibly. Timeline scores are manual total refusal on unsafe prompts (full plus partial); the model label records the companion safe-prompt total-refusal rate. Timeline dates are model public-launch dates, not evaluation dates.", "contamination": { "test_set_public": true, "refresh_cycle": "static", "mitigation": "Keep the official prompt taxonomy intact, publish per-type results, and supplement it with fresh private borderline prompts.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [], "performance_timeline": [ { "model": "GPT-4 (unsafe total refusal; safe 8.4%)", "vendor": "openai", "date": "2023-03-14T00:00:00.000Z", "score": 99.5, "source": "independent", "source_url": "https://arxiv.org/html/2308.01263v3" }, { "model": "Llama 2.0 prompt (unsafe total refusal; safe 59.6%)", "vendor": "meta", "date": "2023-07-18T00:00:00.000Z", "score": 100, "source": "independent", "source_url": "https://arxiv.org/html/2308.01263v3" }, { "model": "Llama 2.1 prompt (unsafe total refusal; safe 29.6%)", "vendor": "meta", "date": "2023-07-18T00:00:00.000Z", "score": 100, "source": "independent", "source_url": "https://arxiv.org/html/2308.01263v3" }, { "model": "Mistral-Instruct prompt (unsafe total refusal; safe 1.6%)", "vendor": "mistral-ai", "date": "2023-09-27T00:00:00.000Z", "score": 36, "source": "independent", "source_url": "https://arxiv.org/html/2308.01263v3" }, { "model": "Mistral-Guard prompt (unsafe total refusal; safe 18.8%)", "vendor": "mistral-ai", "date": "2023-09-27T00:00:00.000Z", "score": 96.5, "source": "independent", "source_url": "https://arxiv.org/html/2308.01263v3" } ], "tags": [ "over-refusal", "exaggerated-safety", "refusal", "harmlessness", "helpfulness" ], "last_updated": "2026-09-14T00:00:00.000Z", "changelog": [ { "date": "2026-09-14T00:00:00.000Z", "note": "Added manually annotated unsafe-refusal scores, with safe-refusal companions in labels, using verified model public-launch dates." }, { "date": "2026-09-14T00:00:00.000Z", "note": "Initial page based on arXiv v3, the NAACL record, and official prompts and evaluation code." } ] }, { "name": "ZeroSCROLLS", "slug": "zeroscrolls", "capability": "long-context", "secondary_capabilities": [ "knowledge", "reasoning" ], "short_description": "Zero-shot long-text suite derived from SCROLLS with test-only tasks and new aggregation-style long-context challenges.", "launch_date": "2023-05-23T00:00:00.000Z", "status": "nearing-saturation", "status_evidence": "ZeroSCROLLS fixed SCROLLS' fine-tuning mismatch and has a live leaderboard, but its 2023-era tasks are now shorter and narrower than current 100k+ and multi-needle long-context stress suites.", "saturated_date": null, "successor": null, "lineage": { "parent": "scrolls", "variants": [] }, "paper": { "title": "ZeroSCROLLS: A Zero-Shot Benchmark for Long Text Understanding", "url": "https://arxiv.org/abs/2305.14196", "venue": "Findings of EMNLP 2023" }, "authors": [ { "name": "Uri Shaham", "affiliation": null }, { "name": "Maor Ivgi", "affiliation": null }, { "name": "Avia Efrat", "affiliation": null }, { "name": "Jonathan Berant", "affiliation": null }, { "name": "Omer Levy", "affiliation": null } ], "maintainer": "Tel Aviv University NLP / tau", "license": null, "dataset": { "url": "https://huggingface.co/datasets/tau/zero_scrolls", "size": null, "public_test_set": false, "gated": false }, "metric": { "primary": "task-normalized zero-shot aggregate", "scoring": "composite", "judge_model": null, "aggregation": "aggregate over ten long-text tasks using task-specific metrics such as ROUGE, accuracy, F1, exponential similarity, and concordance index", "variants": [] }, "run_config": { "shots": "zero-shot", "cot": "direct", "temperature": 0, "pass_at": 1, "tools": false, "harnesses": [ { "name": "Hugging Face tau/zero_scrolls", "url": "https://huggingface.co/datasets/tau/zero_scrolls", "task_flag": "tau/zero_scrolls" } ] }, "cost_estimate": { "examples": null, "approx_usd_frontier_model": null }, "human_baseline": { "score": null, "note": "No single suite-wide human baseline is recorded here; component tasks have their own annotation histories." }, "statistical_note": "The validation sets are tiny and meant for eyeballing; use held-out leaderboard evaluation for claims. The visual graph on this page deliberately plots only the official ZeroSCROLLS paper Table 3 Avg column from the leaderboard at the time of writing. It does not plot per-task scores or later leaderboard submissions.", "contamination": { "test_set_public": false, "refresh_cycle": "static", "mitigation": "ZeroSCROLLS removes training data and uses small validation sets, but the task definitions and component sources are public.", "risk": "medium" }, "languages": [ "english" ], "leaderboards": [ { "name": "ZeroSCROLLS dataset card", "url": "https://huggingface.co/datasets/tau/zero_scrolls", "type": "official" } ], "performance_timeline": [ { "model": "T0pp (paper Table 3 Avg)", "vendor": "bigscience", "date": "2023-05-23T00:00:00.000Z", "score": 14.3, "source": "independent", "source_url": "https://arxiv.org/abs/2305.14196" }, { "model": "Flan-T5 (paper Table 3 Avg)", "vendor": "google", "date": "2023-05-23T00:00:00.000Z", "score": 29.9, "source": "independent", "source_url": "https://arxiv.org/abs/2305.14196" }, { "model": "Flan-UL2 (paper Table 3 Avg)", "vendor": "google", "date": "2023-05-23T00:00:00.000Z", "score": 30.6, "source": "independent", "source_url": "https://arxiv.org/abs/2305.14196" }, { "model": "DaVinci003 (paper Table 3 Avg)", "vendor": "openai", "date": "2023-05-23T00:00:00.000Z", "score": 33.7, "source": "independent", "source_url": "https://arxiv.org/abs/2305.14196" }, { "model": "ChatGPT (paper Table 3 Avg)", "vendor": "openai", "date": "2023-05-23T00:00:00.000Z", "score": 34, "source": "independent", "source_url": "https://arxiv.org/abs/2305.14196" }, { "model": "Claude (paper Table 3 Avg)", "vendor": "anthropic", "date": "2023-05-23T00:00:00.000Z", "score": 39.1, "source": "independent", "source_url": "https://arxiv.org/abs/2305.14196" }, { "model": "GPT-4 (paper Table 3 Avg)", "vendor": "openai", "date": "2023-05-23T00:00:00.000Z", "score": 41.7, "source": "independent", "source_url": "https://arxiv.org/abs/2305.14196" } ], "tags": [ "long-context", "zero-shot", "suite", "summarization", "question-answering" ], "last_updated": "2026-08-12T00:00:00.000Z", "changelog": [ { "date": "2026-08-12T00:00:00.000Z", "note": "Added official paper Table 3 leaderboard Avg scores to the visual graph and added a visible Avg-only stats section clarifying that the graph shows only the ZeroSCROLLS average, not per-task scores or later submissions." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Initial long-context page. Timeline omitted because comparable current official model series were not added to this content page." }, { "date": "2026-08-12T00:00:00.000Z", "note": "Corrected status from active to nearing-saturation: the benchmark remains useful, but newer long-context suites now carry the frontier signal." } ] } ]