[ { "id": "mmlu-pro", "name": "MMLU-Pro", "kind": "acc", "chance": 0.1, "url": "https://github.com/TIGER-AI-Lab/MMLU-Pro" }, { "id": "gpqa-diamond", "name": "GPQA Diamond", "kind": "acc", "chance": 0.25, "url": "https://github.com/idavidrein/gpqa" }, { "id": "hle", "name": "Humanity's Last Exam", "kind": "acc", "chance": 0, "url": "https://lastexam.ai" }, { "id": "aime-2025", "name": "AIME 2025", "kind": "acc", "chance": 0, "url": "https://artofproblemsolving.com/wiki/index.php/2025_AIME_I" }, { "id": "swe-bench-verified", "name": "SWE-bench Verified", "kind": "acc", "chance": 0, "url": "https://www.swebench.com" }, { "id": "aime-2026", "name": "AIME 2026", "kind": "acc", "chance": 0, "url": "https://artofproblemsolving.com/wiki/index.php/2026_AIME_I" }, { "id": "swe-bench-pro", "name": "SWE-bench Pro", "kind": "acc", "chance": 0, "url": "https://scale.com/leaderboard/swe_bench_pro_public" }, { "id": "lmarena-text", "name": "LMArena (text)", "kind": "elo", "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset" }, { "id": "lmarena-image", "name": "LMArena text-to-image", "kind": "elo", "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset", "category": "image" }, { "id": "lmarena-video", "name": "LMArena video", "kind": "elo", "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset", "category": "video" }, { "id": "open-asr-wer", "name": "Open ASR English short-form WER (March 2026)", "kind": "err", "url": "https://github.com/huggingface/open_asr_leaderboard", "category": "stt" }, { "id": "lmarena-webdev", "name": "Arena WebDev (Code Arena)", "kind": "elo", "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset", "category": "llm" }, { "id": "lmarena-video-i2v", "name": "Arena image-to-video", "kind": "elo", "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset", "category": "video" }, { "id": "otis-mock-aime", "name": "OTIS mock AIME 2024–2025", "kind": "acc", "category": "llm", "url": "https://epoch.ai/benchmarks/use-this-data", "chance": 0.001 }, { "id": "frontiermath", "name": "FrontierMath tiers 1–3 v2", "kind": "acc", "category": "llm", "url": "https://epoch.ai/benchmarks/use-this-data" }, { "id": "simpleqa-verified", "name": "SimpleQA Verified", "kind": "acc", "category": "llm", "url": "https://epoch.ai/benchmarks/use-this-data" }, { "id": "open-asr-longform-wer", "name": "Open ASR English long-form WER", "kind": "err", "category": "stt", "url": "https://github.com/huggingface/open_asr_leaderboard" }, { "id": "open-asr-multilingual-wer", "name": "Open ASR multilingual WER", "kind": "err", "category": "stt", "url": "https://github.com/huggingface/open_asr_leaderboard" }, { "id": "livecodebench-v6", "name": "LiveCodeBench v6 (Feb–May 2025)", "kind": "acc", "category": "llm", "url": "https://epoch.ai/benchmarks/use-this-data" }, { "id": "epoch-chess", "name": "Chess Puzzles (Epoch)", "kind": "acc", "category": "llm", "url": "https://epoch.ai/benchmarks/use-this-data", "chance": 0.0496 }, { "id": "epoch-ebr", "name": "EBR-bench (Epoch)", "kind": "acc", "category": "llm", "url": "https://epoch.ai/benchmarks/use-this-data", "chance": 0 }, { "id": "frontiermath-tier-4-v2", "name": "FrontierMath tier 4 v2", "kind": "acc", "category": "llm", "url": "https://epoch.ai/benchmarks/use-this-data", "chance": 0 }, { "id": "math-level-5", "name": "MATH level 5", "kind": "acc", "category": "llm", "url": "https://epoch.ai/benchmarks/use-this-data", "chance": 0 }, { "id": "epoch-mirrorcode", "name": "MirrorCode (Epoch)", "kind": "acc", "category": "llm", "url": "https://epoch.ai/benchmarks/use-this-data", "chance": 0 }, { "id": "epoch-mystery-games", "name": "Mystery Game Puzzles (Epoch)", "kind": "acc", "category": "llm", "url": "https://epoch.ai/benchmarks/use-this-data", "chance": 0.0922 }, { "id": "epoch-furniture", "name": "Furniture Assembly (Epoch)", "kind": "acc", "category": "llm", "url": "https://epoch.ai/benchmarks/use-this-data", "chance": 0 }, { "id": "lmarena-vision", "name": "Arena vision (style controlled)", "kind": "elo", "category": "llm", "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset" }, { "id": "lmarena-document", "name": "Arena document (style controlled)", "kind": "elo", "category": "llm", "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset" }, { "id": "lmarena-search", "name": "Arena search systems (style controlled)", "kind": "elo", "category": "llm", "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset" }, { "id": "scicode", "name": "SciCode", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1" }, { "id": "bbeh-micro", "name": "Big Bench Extra Hard micro avg", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1" }, { "id": "hle-gemma4-report", "name": "HLE", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1" }, { "id": "ifbench", "name": "IFBench", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1" }, { "id": "ifeval-gemma4-report", "name": "IFEval", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1" }, { "id": "mmmlu", "name": "MMMLU", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1", "chance": 0.25 }, { "id": "mrcr-v2-8needle-128k", "name": "MRCR v2 8-needle, 128k", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1" }, { "id": "terminalbench-hard-gemma4", "name": "Terminal-Bench Hard", "kind": "acc", "category": "llm", "url": "https://github.com/NVIDIA-NeMo/Evaluator/blob/e77dfb31fd1538080cbfb5a50f0ba0e4cb8f253f/src/nemo_evaluator/benchmarks/terminal_bench_hard.py", "chance": 0, "description": "Hard-task subset of the original Terminal-Bench (terminal-bench-core, commit 74221fb; 47 tasks, or the 44-task Artificial Analysis split) run with the Terminus 2 agent, as used in the Artificial Analysis Intelligence Index. Pass rate. Task count, repeats and reasoning setting per note. Not Terminal-Bench 2.0/2.1/3.0/4.0, and not the 100-task \"TBH\" used in some papers." }, { "id": "tau2-airline-gemma4", "name": "Tau2 – airline", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1" }, { "id": "tau2-retail-gemma4", "name": "Tau2 – retail", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1" }, { "id": "mmmu-pro", "name": "MMMU Pro", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1", "chance": 0.126, "description": "Chance 0.126: published random baseline (https://arxiv.org/abs/2409.02813; MMMU-Pro paper, Table 1: 'Random Choice 24.9 | 12.8 | 12.4' for Standard (4 Opts) | Standard (10 Opts) | Vision. Section 3: 'The overall performance score for M)." }, { "id": "math-vision", "name": "MATH-Vision", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1", "chance": 0.0717, "description": "Chance 0.0717: published random baseline (https://arxiv.org/abs/2402.14804; MATH-Vision paper, results table: 'Random Chance 7.17' (Overall); text: 'the comparison of models’ performance with random chance (7.17% overall)'.)." }, { "id": "medxpertqa-mm", "name": "MedXPertQA MM", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1", "chance": 0.2 }, { "id": "infographicvqa", "name": "InfographicVQA", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1" }, { "id": "omnidocbench-1-5", "name": "OmniDocBench 1.5 ↓", "kind": "err", "category": "llm", "url": "https://arxiv.org/html/2607.02770v1" }, { "id": "tau2-verified-gpt51-user", "name": "τ²-Bench-Verified (GPT-5.1 user)", "kind": "acc", "category": "llm", "url": "https://github.com/amazon-agi/tau2-bench-verified/blob/864350a8971a8f8ee9e7b8472e2edc380a806b0c/README.md" }, { "id": "massive-agents-ast-10k", "name": "MASSIVE-Agents AST (10k, zero-shot)", "kind": "acc", "category": "llm", "url": "https://aclanthology.org/2025.findings-emnlp.1099.pdf" }, { "id": "aider-polyglot", "name": "Aider polyglot (225 exercises, two attempts)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml" }, { "id": "bfcl-v4", "name": "BFCL V4 (December 2025 archive)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-12-16/score/data_overall.csv" }, { "id": "click-korean-kim", "name": "CLIcK Korean (Daekeun Kim)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md" }, { "id": "cruxeval-zeroeval", "name": "CRUXEval (ZeroEval, 800 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/crux.summary.json" }, { "id": "evalperf-correctness-20240328", "name": "EvalPerf correctness (118 tasks, 20240328)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/evalplus/evalplus.github.io/blob/87418456d3a8c2bc6265f24ff91d2dd0e7993f4b/results/evalperf/Qwen--Qwen2.5-7B-Instruct_vllm_temp_1.0_evalperf_results.brief.json" }, { "id": "gsm8k-zeroeval", "name": "GSM8K (ZeroEval, zero-shot)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/gsm.summary.json" }, { "id": "haerae-korean-kim", "name": "HAE-RAE 1.0 (Daekeun Kim)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md" }, { "id": "kmmlu-hard-kim", "name": "KMMLU-HARD (Daekeun Kim, zero-shot)", "kind": "acc", "chance": 0.25, "category": "llm", "url": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/PREVIOUS_RESULTS.md" }, { "id": "kmmlu-pro-kim", "name": "KMMLU-Pro (Daekeun Kim)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md" }, { "id": "kobalt-700-kim", "name": "KoBALT-700 (Daekeun Kim)", "kind": "acc", "chance": 0.1, "category": "llm", "url": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md" }, { "id": "mmlu-redux-zeroeval", "name": "MMLU-Redux (ZeroEval, 2,778 questions)", "kind": "acc", "chance": 0.25, "category": "llm", "url": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/mmlu-redux.summary.json" }, { "id": "musr-ko-kim", "name": "MuSR Korean (Daekeun Kim)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/daekeun-ml/evaluate-llm-on-korean-dataset/blob/106ed712b5b88f0e531d5fa503bfed76ac31396e/docs/RESULTS.md" }, { "id": "zebralogic-zeroeval", "name": "ZebraLogic (ZeroEval, 1,000 puzzles)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/zebra-grid.summary.json" }, { "id": "agieval-en", "name": "AGIEval English", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2512.13961v2", "description": "Aggregate across nine English AGIEval subtests. No subtest rows are also added." }, { "id": "aime-2024", "name": "AIME 2024", "kind": "acc", "category": "llm", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md", "description": "Pass@1 / mean single-response accuracy, not majority-vote cons@64." }, { "id": "alpacaeval2-lc", "name": "AlpacaEval 2 length-controlled", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2512.13961v2", "description": "Length-controlled win rate; judge/version and decoding depend on source." }, { "id": "arc-challenge", "name": "ARC Challenge", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Accuracy on ARC Challenge; not ARC Easy.", "chance": 0.25 }, { "id": "arena-hard-v2", "name": "Arena-Hard v2 aggregate", "kind": "acc", "category": "llm", "url": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md", "description": "Aggregate of hard prompts and creative writing. Component rows not also imported." }, { "id": "bbh", "name": "BIG-Bench Hard", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Aggregate across the 23 BBH tasks; no individual task rows are also added. Chance 0.257: published random baseline (https://arxiv.org/abs/2210.09261; Suzgun et al. 2022 (BIG-Bench Hard), Table 3, column 'Srivastava et al. (2022) Random': 'All Tasks (avg) 25.7' (NLP Task avg 29.5, Algorithmic Task avg 21.2); t).", "chance": 0.257 }, { "id": "bfcl-v3", "name": "BFCL v3", "kind": "acc", "category": "llm", "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md", "description": "Berkeley Function Calling Leaderboard v3 aggregate. No aggregate-plus-subcomponent duplication." }, { "id": "boolq", "name": "BoolQ", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Boolean question-answering accuracy.", "chance": 0.5 }, { "id": "gpqa-unspecified", "name": "GPQA (subset unspecified)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Source only labels GPQA. Diamond identity is not inferred. Keep out of a Diamond ranking.", "chance": 0.25 }, { "id": "gsm8k", "name": "GSM8K", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Grade-school mathematics accuracy; shots and CoT setup in result notes." }, { "id": "hellaswag", "name": "HellaSwag", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Commonsense completion accuracy; source protocol retained.", "chance": 0.25 }, { "id": "hmmt-2025-edition-unspecified", "name": "HMMT 2025 (edition unspecified)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2601.02346v1", "description": "Source names HMMT25 without resolving the February or November contest edition." }, { "id": "hmmt-feb-2025", "name": "HMMT February 2025", "kind": "acc", "category": "llm", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md", "description": "February 2025 HMMT math contest; not November or other HMMT editions." }, { "id": "humaneval", "name": "HumanEval", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md", "description": "Original HumanEval pass@1; distinct from HumanEval+." }, { "id": "humaneval-plus", "name": "HumanEval+", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2512.13961v2", "description": "EvalPlus expanded-test pass@1; distinct from original HumanEval." }, { "id": "ifbench-prompt", "name": "IFBench prompt-level", "kind": "acc", "category": "llm", "url": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md", "description": "Prompt-level IFBench instruction following; distinct from IFEval." }, { "id": "ifbench-unspecified", "name": "IFBench (aggregation unspecified)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2512.13961v2", "description": "Published IFBench score where card/report does not explicitly identify scoring aggregation." }, { "id": "ifeval-instruction-strict", "name": "IFEval instruction-level strict", "kind": "acc", "category": "llm", "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/blob/6533e8de2c68e4536bf7c411d7a3ce5734111476/README.md", "description": "Strict instruction-level accuracy, distinct from prompt-level and loose scores." }, { "id": "ifeval-prompt-unspecified-strictness", "name": "IFEval prompt-level (strictness unspecified)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f/README.md", "description": "Prompt-level score explicitly identified, but strict versus loose checking not specified." }, { "id": "ifeval-unspecified", "name": "IFEval (aggregation unspecified)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md", "description": "Published IFEval score; source does not identify instruction/prompt or strict/loose aggregation." }, { "id": "livecodebench-unversioned", "name": "LiveCodeBench (version unspecified)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md", "description": "Published pass@1; source does not specify version/date window. Separate from every versioned score." }, { "id": "livecodebench-v3", "name": "LiveCodeBench v3", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2512.13961v2", "description": "LiveCodeBench v3 pass@1; distinct from v4, v5, and v6 windows." }, { "id": "livecodebench-v5-202407-202412", "name": "LiveCodeBench v5 (Jul–Dec 2024)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md", "description": "Source-specified v5 evaluation window 2024-07 through 2024-12." }, { "id": "livecodebench-v6-202408-202505", "name": "LiveCodeBench v6 (Aug 2024–May 2025)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B/blob/6e8885a6ff5c1dc5201574c8fd700323f23c25fa/README.md", "description": "Explicit full 2408–2505 window; NOT the Feb–May 2025 v6 slice already in the shared dataset." }, { "id": "livecodebench-v6-window-unspecified", "name": "LiveCodeBench v6 (window unspecified)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2601.02346v1", "description": "v6 explicitly specified but date filter not stated. Do not silently merge with a date-filtered slice." }, { "id": "longbench-v2", "name": "LongBench v2", "kind": "acc", "category": "llm", "url": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md", "description": "Long-context reasoning benchmark version 2, distinct from LongBench v1.", "chance": 0.25 }, { "id": "math", "name": "MATH", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Full MATH test accuracy; prompt and sample setup retained in each result." }, { "id": "math-500", "name": "MATH-500", "kind": "acc", "category": "llm", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md", "description": "Accuracy on the 500-problem MATH test subset. Do not combine with full MATH." }, { "id": "math-subset-unspecified", "name": "MATH (subset unspecified)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md", "description": "Source only identifies MATH; full 5000-question test versus MATH-500 is unresolved. Not merged with either." }, { "id": "mbpp-plus", "name": "MBPP+", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2512.13961v2", "description": "EvalPlus expanded-test MBPP pass@1." }, { "id": "mgsm", "name": "MGSM", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Multilingual grade-school mathematics accuracy." }, { "id": "mmlu", "name": "MMLU", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Original MMLU aggregate across 57 subjects; distinct from MMLU-Pro/Redux.", "chance": 0.25 }, { "id": "mmlu-redux", "name": "MMLU-Redux", "kind": "acc", "category": "llm", "url": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md", "description": "MMLU-Redux aggregate; not original MMLU or MMLU-Pro.", "chance": 0.25 }, { "id": "multilingual-mmlu", "name": "Multilingual MMLU (Microsoft)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Microsoft card’s multilingual MMLU aggregate; language set not specified, separate from original MMLU." }, { "id": "openbookqa", "name": "OpenBookQA", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "OpenBookQA test accuracy.", "chance": 0.25 }, { "id": "piqa", "name": "PIQA", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Physical commonsense question-answering accuracy.", "chance": 0.5 }, { "id": "popqa", "name": "PopQA", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2512.13961v2", "description": "PopQA factual QA using source-described answer matching." }, { "id": "scicode-main", "name": "SciCode main problems", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2601.02346v1", "description": "End-to-end main scientific coding problems; subproblem score is deliberately not also added." }, { "id": "scicode-subtask", "name": "SciCode subproblems", "kind": "acc", "category": "llm", "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/blob/bf77c3174f68ad409e1c2aa60daeb46e32d1c606/README.md", "description": "Subproblem score explicitly labelled by source. Distinct from main problems." }, { "id": "simpleqa", "name": "SimpleQA", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/phi-4/blob/2db69c1c3e91a05d2c64a3185acfbaf36f744e25/README.md", "description": "Correct-answer fraction on SimpleQA; not SimpleQA Verified or search-augmented SimpleQA." }, { "id": "social-iqa", "name": "Social IQA", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Social commonsense accuracy.", "chance": 0.3333 }, { "id": "swe-bench-live", "name": "SWE-bench Live", "kind": "acc", "category": "llm", "url": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md", "description": "Resolved fraction; exact snapshot not specified in source chart." }, { "id": "swe-bench-multilingual", "name": "SWE-bench Multilingual", "kind": "acc", "category": "llm", "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/2dc98e2afe4face0e4ce40972a915c45368bd34a/README.md", "description": "Resolved fraction on SWE-bench Multilingual; scaffold retained." }, { "id": "tau-bench-airline", "name": "τ-bench Airline", "kind": "acc", "category": "llm", "url": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md", "description": "Original τ-bench Airline, not τ²-bench." }, { "id": "tau-bench-retail", "name": "τ-bench Retail", "kind": "acc", "category": "llm", "url": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/b2cff646eb4bb1d68355c01b18ae02e7cf42d120/README.md", "description": "Original τ-bench Retail, not τ²-bench." }, { "id": "tau2-bench", "name": "τ²-bench aggregate", "kind": "acc", "category": "llm", "url": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md", "description": "Average of the Airline, Retail and Telecom domains. Only aggregate retained when provided." }, { "id": "terminal-bench-2", "name": "Terminal-Bench 2.0", "kind": "acc", "category": "llm", "url": "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B/blob/6327cdbcf907e1c7cec9cb29fb6e6cebdf8feaf7/README.md", "description": "Terminal-Bench version 2.0. Agent scaffold, token limits, and subset retained in notes." }, { "id": "truthfulqa-mc2", "name": "TruthfulQA MC2", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Probability mass assigned to true answers; not MC1 or generation accuracy." }, { "id": "winogrande", "name": "WinoGrande", "kind": "acc", "category": "llm", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md", "description": "Pronoun/coreference reasoning accuracy.", "chance": 0.5 }, { "id": "zebralogic", "name": "ZebraLogic", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2512.13961v2", "description": "Logical-puzzle accuracy under the report’s JSON scorer." }, { "id": "ttsds-v1-default", "name": "TTSDS v1 · default four-factor score", "category": "tts", "kind": "acc", "url": "https://huggingface.co/datasets/ttsds/results/blob/111ccecaa50aec4ca1247c5f5d95c2058f6f2946/results.csv", "metric": "distribution_similarity_score", "setup": "English reference-based synthesis on ttsds/speaker_text_pairs. Mean within general/prosody/speaker/intelligibility, then unweighted mean across those four factors. Environment excluded exactly as the original default UI. Distance to real/noise reference distributions; not recognition accuracy or MOS.", "aggregation": "mean of category means for enum values 1,2,4,6 in ttsds 0.0.2", "score_unit": "source 0–100 divided by 100", "caveats": [ "Model submissions name versions but do not pin weight commits. Inference details sparse.", "Snapshot date is dataset update, not the generation date.", "Different metrics choose different nearest real/noise reference datasets by design." ] }, { "id": "emergent-tts-6cat-stable-wer", "name": "EmergentTTS · 6-category WER", "category": "tts", "kind": "err", "url": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md", "metric": "word_error_rate", "setup": "1,645 English test cases in six categories: emotion, paralinguistics, foreign words, syntactic complexity, complex pronunciation and questions. Current stable gemini-2.5-pro judge, thinking_budget=256; baseline gpt-4o-mini-tts Alloy. Source update [8-10-25]. WER uses Whisper-large-v3. Voice and strong-prompting choice recorded per result." }, { "id": "emergent-tts-6cat-stable-preference", "name": "EmergentTTS · 6-category preference", "category": "tts", "kind": "acc", "url": "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/README.md", "metric": "judge_win_rate", "setup": "1,645 English test cases in six categories: emotion, paralinguistics, foreign words, syntactic complexity, complex pronunciation and questions. Current stable gemini-2.5-pro judge, thinking_budget=256; baseline gpt-4o-mini-tts Alloy. Source update [8-10-25]. WER uses Whisper-large-v3. Voice and strong-prompting choice recorded per result.", "caveats": [ "LLM preference against one fixed baseline, not human MOS.", "Different voices and prompting configurations limit model-only attribution." ] }, { "id": "voxtral-tts-minimax9-wer", "name": "MiniMax TTS · 9-language WER (Voxtral ASR)", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2603.25551v2", "metric": "word_error_rate", "setup": "Zero-shot voice cloning, Table 3. WER transcriber Voxtral Mini Transcribe v2; speaker similarity ECAPA-TDNN embedding cosine. API build dates and exact generation seeds not provided. Equal-language mean derived across Arabic, German, English, Spanish, French, Hindi, Italian, Dutch, Portuguese. Do not combine with the 24-language MiniMax suite." }, { "id": "voxtral-tts-minimax9-speaker", "name": "MiniMax TTS · 9-language speaker similarity", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2603.25551v2", "metric": "speaker_embedding_cosine", "setup": "Zero-shot voice cloning, Table 3. WER transcriber Voxtral Mini Transcribe v2; speaker similarity ECAPA-TDNN embedding cosine. API build dates and exact generation seeds not provided. Same nine-language macro mean; this is voice similarity, not accuracy." }, { "id": "voxtral-tts-seed-reported-wer", "name": "Seed-TTS · report split unspecified (Voxtral ASR)", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2603.25551v2", "metric": "word_error_rate", "setup": "Zero-shot voice cloning, Table 3. WER transcriber Voxtral Mini Transcribe v2; speaker similarity ECAPA-TDNN embedding cosine. API build dates and exact generation seeds not provided. Report labels row Seed TTS without explicit split/language; compare only within this report.", "caveats": [ "Seed-TTS split/language not stated; intentionally separate from canonical test-en." ] }, { "id": "voxtral-tts-seed-reported-speaker", "name": "Seed-TTS · report split unspecified, speaker similarity", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2603.25551v2", "metric": "speaker_embedding_cosine", "setup": "Zero-shot voice cloning, Table 3. WER transcriber Voxtral Mini Transcribe v2; speaker similarity ECAPA-TDNN embedding cosine. API build dates and exact generation seeds not provided. Report does not identify Seed-TTS split/language; kept separate from other Seed metrics." }, { "id": "seed-tts-en-whisper-v3-wer", "name": "Seed-TTS test-en · Whisper large-v3 WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2603.08823v2", "metric": "word_error_rate", "setup": "Zero-shot voice cloning on standard Seed-TTS-Eval test-en. Whisper-large-v3 transcripts. Different papers/runs retain individual configurations and provenance; no cross-paper remeasurement implied." }, { "id": "seed-tts-zh-paraformer-cer", "name": "Seed-TTS test-zh · Paraformer CER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2603.08823v2", "metric": "character_error_rate", "setup": "Zero-shot voice cloning on standard Seed-TTS-Eval test-zh. Paraformer Chinese transcripts. Papers label the compatibility column WER, but Chinese metric uses character error rate." }, { "id": "fish-long-tts-en-wer", "name": "Long-TTS-Eval · Fish truncated English WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2603.08823v2", "metric": "word_error_rate", "setup": "Long-TTS-Eval six content categories, Seed-TTS reference voices. Fish context-limited subset truncates text at sentence boundaries, 74–1211 English tokens, mean 760, threshold 939 with ±30% perturbation. Whisper-large-v3 on non-overlapping 28-second chunks, transcripts concatenated. Not the unmodified Long-TTS-Eval set." }, { "id": "fish-long-tts-zh-cer", "name": "Long-TTS-Eval · Fish truncated Chinese CER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2603.08823v2", "metric": "character_error_rate", "setup": "Same modified Long-TTS-Eval design; Chinese 32–1146 tokens, mean 589, threshold 884 with ±30% perturbation. Paraformer-zh, non-overlapping 28-second chunks. Full sample counts/seeds not supplied." }, { "id": "voxtral-rt-2026-en-short", "name": "Voxtral report · English short-form 10-set WER", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2602.11298v1", "metric": "word_error_rate", "setup": "Voxtral Realtime Table 3 and Appendix A.1; vendor-run evaluation. Reported macro averages, not corpus-weighted error rates. API dated build identifiers and detailed text normalization are not supplied; treat as the February 2026 report snapshot. LibriSpeech clean/other, GigaSpeech, VoxPopuli, Switchboard, CallHome, CHiME4, SPGISpeech, TED-LIUM, Earnings22. Different suite from Open ASR leaderboard eight-set aggregate." }, { "id": "voxtral-rt-2026-en-long", "name": "Voxtral report · English long-form 4-set WER", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2602.11298v1", "metric": "word_error_rate", "setup": "Voxtral Realtime Table 3 and Appendix A.1; vendor-run evaluation. Reported macro averages, not corpus-weighted error rates. API dated build identifiers and detailed text normalization are not supplied; treat as the February 2026 report snapshot. Meanwhile, TED-LIUM long-form, Earnings21 and Earnings22 calls segmented into ten-minute variants." }, { "id": "voxtral-rt-2026-fleurs13", "name": "Voxtral report · FLEURS 13-language error rate", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2602.11298v1", "metric": "mixed_word_character_error_rate", "setup": "Voxtral Realtime Table 3 and Appendix A.1; vendor-run evaluation. Reported macro averages, not corpus-weighted error rates. API dated build identifiers and detailed text normalization are not supplied; treat as the February 2026 report snapshot. Languages ar,de,en,es,fr,hi,it,ja,ko,nl,pt,ru,zh; Japanese/Chinese CER, other languages WER. Do not label a homogeneous multilingual WER." }, { "id": "voxtral-rt-2026-mcv12", "name": "Voxtral report · Common Voice 12-language error rate", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2602.11298v1", "metric": "mixed_word_character_error_rate", "setup": "Voxtral Realtime Table 3 and Appendix A.1; vendor-run evaluation. Reported macro averages, not corpus-weighted error rates. API dated build identifiers and detailed text normalization are not supplied; treat as the February 2026 report snapshot. Languages de,en,es,fr,hi,it,ja,ko,nl,pt,ru,zh. Japanese/Chinese CER, other languages WER. Arabic excluded by paper because all systems exceed 45%. Common Voice release not specified." }, { "id": "gigaspeechbench-arabic7-macro", "name": "GigaSpeechBench · 7 Arabic dialect WER mean", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2606.28884v1", "metric": "word_error_rate", "setup": "Derived equal-dialect mean of Table 2 columns IRQ,DZA,ARE,EGY,MAR,SAU,SYR. Real-world spontaneous speech; low-resource sets about 20h each; utterances ≤0.5s filtered. Original per-dialect scores retained as provenance, not additional results. API build snapshots and full normalization not specified." }, { "id": "gigaspeechbench-en-accents6-macro", "name": "GigaSpeechBench · 6 English accents WER mean", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2606.28884v1", "metric": "word_error_rate", "setup": "Derived equal-column mean of Table 5: CHN-EN,IDN-EN,JPN-EN,PHL-EN,SCT-EN,SGP-EN, approximately 10h each. Preserve source paper code IDN-EN: repository README instead says IND-EN; do not silently relabel the accent.", "caveats": [ "Paper/README IDN-EN versus IND-EN naming discrepancy." ] }, { "id": "gigaspeechbench-en-domain12-bwer", "name": "GigaSpeechBench · 12 English domains entity B-WER mean", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2606.28884v1", "metric": "entity_biased_word_error_rate", "setup": "Derived equal-domain mean of Table 8. AGR,AIT,ART,BIO,ECM,ENG,ENT,FIN,HUM,LAW,MED,MIL English columns. B-WER evaluates domain-specific biasing/entity words, not ordinary all-word WER. Table reports roughly 10–11.6h valid reference audio per domain." }, { "id": "stt-ru-vakovalskii-fleurs", "name": "Russian STT · fleurs corpus WER", "category": "stt", "kind": "err", "url": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/README.md", "metric": "word_error_rate", "setup": "August 2026 direct API/local NeMo runs. google/fleurs, ru_ru test. Corpus micro WER: sum word edits / sum reference words. Lowercase, ё→е, remove punctuation, collapse whitespace; no number normalization. Deepgram language=ru, punctuate=true, smart_format=false. NVIDIA Parakeet on RTX4090. Successful rows only; Nova-2 audiobook n=295 vs296 others. CI from source published utterance bootstrap.", "sample_count": 775, "evidence_tier": "independent_raw_hypotheses", "caveats": [ "Russian only; no claim about overall multilingual ability.", "README says 20,000 bootstrap replications; reproduction command uses 50,000; do not infer exact CI run count." ] }, { "id": "stt-ru-vakovalskii-rulibrispeech", "name": "Russian STT · rulibrispeech corpus WER", "category": "stt", "kind": "err", "url": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/README.md", "metric": "word_error_rate", "setup": "August 2026 direct API/local NeMo runs. bond005/rulibrispeech test. Corpus micro WER: sum word edits / sum reference words. Lowercase, ё→е, remove punctuation, collapse whitespace; no number normalization. Deepgram language=ru, punctuate=true, smart_format=false. NVIDIA Parakeet on RTX4090. Successful rows only; Nova-2 audiobook n=295 vs296 others. CI from source published utterance bootstrap.", "sample_count": 296, "evidence_tier": "independent_raw_hypotheses", "caveats": [ "Russian only; no claim about overall multilingual ability.", "README says 20,000 bootstrap replications; reproduction command uses 50,000; do not infer exact CI run count." ] }, { "id": "stt-ru-vakovalskii-podlodka", "name": "Russian STT · podlodka corpus WER (20-clip pilot)", "category": "stt", "kind": "err", "url": "https://github.com/vakovalskii/stt-ru-benchmark/blob/387548b8d9fdf51791a837b0c42059d2c2ebcaa9/README.md", "metric": "word_error_rate", "setup": "August 2026 direct API/local NeMo runs. bond005/podlodka_speech test, first 20 samples. Corpus micro WER: sum word edits / sum reference words. Lowercase, ё→е, remove punctuation, collapse whitespace; no number normalization. Deepgram language=ru, punctuate=true, smart_format=false. NVIDIA Parakeet on RTX4090. Successful rows only; Nova-2 audiobook n=295 vs296 others. CI from source published utterance bootstrap.", "sample_count": 20, "evidence_tier": "pilot", "caveats": [ "Podlodka subset is tiny.", "README says 20,000 bootstrap replications; reproduction command uses 50,000; do not infer exact CI run count." ] }, { "id": "memo20-live-ru-full-median", "name": "Voice memos · RU 10-clip pilot median WER", "category": "stt", "kind": "err", "url": "https://github.com/Smolevich/stt-benchmarks/blob/b6349a817a84cb0d74369b02ea951f1d6112f4f2/results/summary.csv", "metric": "median_utterance_word_error_rate", "setup": "10 live voice memos per language, one speaker on iPhone. Full set includes digit/name stress cases. Median across 30 utterance-run WERs (10 clips ×3), not corpus WER and not 30 independent clips. Deepgram explicit language=ru, smart_format=true, punctuate=true. Do not compare directly to repo headline clean eight-clip means or to corpus-WER suites.", "sample_count": 10, "evidence_tier": "small_private_audio_pilot", "caveats": [ "Private audio prevents independent reproduction.", "No number-word normalization; formatting differences materially affect WER." ] }, { "id": "memo20-live-en-full-median", "name": "Voice memos · EN 10-clip pilot median WER", "category": "stt", "kind": "err", "url": "https://github.com/Smolevich/stt-benchmarks/blob/b6349a817a84cb0d74369b02ea951f1d6112f4f2/results/summary.csv", "metric": "median_utterance_word_error_rate", "setup": "10 live voice memos per language, one speaker on iPhone. Full set includes digit/name stress cases. Median across 30 utterance-run WERs (10 clips ×3), not corpus WER and not 30 independent clips. Deepgram explicit language=en, smart_format=true, punctuate=true. Do not compare directly to repo headline clean eight-clip means or to corpus-WER suites.", "sample_count": 10, "evidence_tier": "small_private_audio_pilot", "caveats": [ "Private audio prevents independent reproduction.", "No number-word normalization; formatting differences materially affect WER." ] }, { "id": "cn-newstts-v01-strict", "name": "CN-NewsTTS v0.1 · strict pronunciation accuracy", "category": "tts", "kind": "acc", "url": "https://github.com/Jayden-X-L/cn-news-tts-bench/blob/5cd2dcf0896f2391b877317221892a24066588bb/artifacts/scores/leaderboard.csv", "metric": "target_pronunciation_accuracy", "setup": "Raw-input Chinese news, public split 800 utterances / 1008 targets, 16 optional targets excluded =>992 evaluable. Correct requires ≥2 of 3 ASR votes (MiMo API ASR, SenseVoiceSmall, Paraformer-zh); unknowns retained in denominator. No external rewriting, normalization frontend or SSML. Invoked 2026-06-22, canonical 24kHz mono. Narrow pronunciation-risk task, not MOS or sentence WER.", "sample_count": 800, "target_count": 992, "caveats": [ "ASR proxy can mask or invent errors.", "Adjacent ranks Google/MiniMax, MiniMax/Aliyun and MiMo/AWS not separated by source paired cluster-bootstrap intervals.", "Some source voice IDs are only descriptive; preserved verbatim, not guessed." ] }, { "id": "geneval", "name": "GenEval", "category": "image", "kind": "acc", "chance": 0, "url": "https://github.com/djghosh13/geneval", "task": "text-to-image", "setup": { "url": "https://github.com/djghosh13/geneval", "version": "Original 553-prompt GenEval", "metric": "Overall macro-average of six task scores, not percent of all prompts; standard evaluator; original generation recipe uses four images per prompt.", "comparability": "Report-specific generation settings and prompt enhancement must be retained per result." }, "reported_scale": "0–1", "benchmark_family": "geneval" }, { "id": "dpg-bench", "name": "DPG-Bench", "category": "image", "kind": "acc", "chance": 0, "url": "https://github.com/TencentQQGYLab/ELLA", "task": "text-to-image", "setup": { "url": "https://github.com/TencentQQGYLab/ELLA", "version": "Original DPG-Bench, 1,065 dense prompts", "metric": "Overall dependency-aware VQA score; report 0–100 divided by 100.", "comparability": "Do not substitute mean of Global/Entity/Attribute/Relation/Other for published overall." }, "reported_scale": "0–100", "benchmark_family": "dpg-bench" }, { "id": "tiif-testmini-short", "name": "TIIF-Bench testmini (short prompts)", "category": "image", "kind": "acc", "chance": 0, "url": "https://github.com/A113N-W3I/TIIF-Bench", "task": "text-to-image", "setup": { "url": "https://github.com/A113N-W3I/TIIF-Bench", "version": "Original TIIF-Bench testmini, not v2", "metric": "Overall, short-prompt testmini subset", "source_protocol": "Mage-Flow Table 10; original evaluation protocol; distinct short/long scores." }, "reported_scale": "0–100", "benchmark_family": "tiif" }, { "id": "tiif-testmini-long", "name": "TIIF-Bench testmini (long prompts)", "category": "image", "kind": "acc", "chance": 0, "url": "https://github.com/A113N-W3I/TIIF-Bench", "task": "text-to-image", "setup": { "url": "https://github.com/A113N-W3I/TIIF-Bench", "version": "Original TIIF-Bench testmini, not v2", "metric": "Overall, long-prompt testmini subset", "source_protocol": "Mage-Flow Table 10; original evaluation protocol; distinct short/long scores." }, "reported_scale": "0–100", "benchmark_family": "tiif" }, { "id": "oneig-en", "name": "OneIG-Bench (EN)", "category": "image", "kind": "acc", "chance": 0, "url": "https://github.com/OneIG-Bench/OneIG-Benchmark", "task": "text-to-image", "setup": { "url": "https://github.com/OneIG-Bench/OneIG-Benchmark", "version": "OneIG-Bench EN split", "metric": "Published Overall combining alignment, text, reasoning, style and diversity; retained as a fraction." }, "reported_scale": "0–1", "benchmark_family": "oneig" }, { "id": "oneig-cn", "name": "OneIG-Bench (CN)", "category": "image", "kind": "acc", "chance": 0, "url": "https://github.com/OneIG-Bench/OneIG-Benchmark", "task": "text-to-image", "setup": { "url": "https://github.com/OneIG-Bench/OneIG-Benchmark", "version": "OneIG-Bench CN split", "metric": "Published Overall combining alignment, text, reasoning, style and diversity; retained as a fraction." }, "reported_scale": "0–1", "benchmark_family": "oneig" }, { "id": "cvtg-2k-word-accuracy", "name": "CVTG-2K word accuracy", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2607.19064v2#S6.T11", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2607.19064v2#S6.T11", "metric": "Published Avg word accuracy across multi-region English text generation; use Avg, not NED or CLIP." }, "reported_scale": "0–1", "benchmark_family": "cvtg-2k-word-accuracy" }, { "id": "longtext-en", "name": "LongText-Bench (EN)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2607.19064v2#S6.T8", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2607.19064v2#S6.T8", "metric": "Long-form text rendering accuracy; distinct English and Chinese tracks." }, "reported_scale": "0–1", "benchmark_family": "longtext" }, { "id": "longtext-cn", "name": "LongText-Bench (CN)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2607.19064v2#S6.T8", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2607.19064v2#S6.T8", "metric": "Long-form text rendering accuracy; distinct English and Chinese tracks." }, "reported_scale": "0–1", "benchmark_family": "longtext" }, { "id": "humanscore-t2v", "name": "HumanScore (T2V)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2604.20157v1#S3.T1", "task": "text-to-video", "setup": { "url": "https://arxiv.org/html/2604.20157v1#S3.T1", "version": "HumanScore v1", "prompts": "51 motion types × gentle/intense = 102 prompts; 17 easy/medium/hard motion types each.", "metric": "Published Overall of anatomy, kinematic and kinetic correctness; six biomechanical submetrics.", "limitations": "Motion types filtered using candidate generators; exact API snapshot, sampler and resolution not reported for every model." }, "reported_scale": "0–100", "benchmark_family": "humanscore-t2v" }, { "id": "rapidata-physics-i2v-reference", "name": "Rapidata physics realism (I2V, with reference)", "category": "video", "kind": "elo", "url": "https://huggingface.co/datasets/Rapidata/world-model-physics", "task": "image-to-video", "setup": { "url": "https://huggingface.co/datasets/Rapidata/world-model-physics", "metric": "Published Bradley–Terry Elo with real continuation shown; 95% CI half-width retained.", "protocol": "65 Physics-IQ scenarios; same switch frame and scene text; four ~5s generations per scenario; ≥5 votes per comparison.", "limitations": "Dataset card states 21,125 comparisons while current dataset viewer reports another count; extract only pinned card scores. Do not mix with blind or overall ratings." }, "reported_scale": "Elo", "benchmark_family": "rapidata-physics-i2v-reference" }, { "id": "rapidata-camera-i2v", "name": "Rapidata camera movement (I2V)", "category": "video", "kind": "elo", "url": "https://huggingface.co/datasets/Rapidata/camera-movement", "task": "image-to-video", "setup": { "url": "https://huggingface.co/datasets/Rapidata/camera-movement", "metric": "Published Movement Elo; humans compare generated camera movement to a reference clip.", "protocol": "600 prompts; 22 stills; 30 movements; 8,400 clips; 281,738 votes. Commercial models nominally 1080p/5s. Veo3.1 uses 6s, MiniMax H3 uses upscaled 2K tier, Gemini Omni Flash1.1 crops/pads some inputs; see per-row evidence.", "limitations": "Only exact named commercial model versions included." }, "reported_scale": "Elo", "benchmark_family": "rapidata-camera-i2v" }, { "id": "physics-iq-verified-i2v-bpp", "name": "Physics-IQ Verified (I2V, best-practice prompts)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2606.18943v1#A5.T4", "task": "image-to-video", "setup": { "url": "https://arxiv.org/html/2606.18943v1#A5.T4", "version": "Verified ground truth + Verified metric + best-practice templated prompts (bpp)", "metric": "Mean over four generation runs; standard deviation retained, not a confidence interval.", "protocol": "66 scenarios × 3 starting times = 198 videos per complete I2V run; four runs; verified ground truth, verified metric and best-practice prompts. Cosmos3-Nano does not use LLM/VLM prompt upsampling.", "limitations": "Do not mix original metric, original ground truth, original prompts (op), v2v, or best-of-N." }, "reported_scale": "0–100", "benchmark_family": "physics-iq-verified-i2v-bpp" }, { "id": "vgi-bench-i2v", "name": "VGI-Bench visual reasoning (I2V)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2608.19583v3#S3.T2", "task": "image-to-video", "setup": { "url": "https://arxiv.org/html/2608.19583v3#S3.T2", "version": "Report v3, primary video branch; fixed half-instance sample across full task taxonomy", "metric": "Published Overall final score (rubric × completeness); not strict success rate or adapted image-branch success.", "protocol": "I2V; half the instances in each task/difficulty chosen with fixed seed; Gemini-3-Flash VLM judge; per-model configurations in Appendix B.1/Table 10.", "limitations": "Keep I2V checkpoints separate from T2V variants." }, "reported_scale": "0–100", "benchmark_family": "vgi-bench-i2v" }, { "id": "qwen-image-bench-v1", "name": "Qwen-Image-Bench (Q-Judger, v1)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2605.28091v1#S4.T2", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2605.28091v1#S4.T2", "version": "Original 18-model evaluation, report v1", "metric": "Overall score on 0–100 scale, aggregated from 56 rubric facets; Q-Judger based on Qwen3.6-27B. Not Appendix human scores.", "protocol": "1,000 stratified creation prompts, five pillars; report Table 2.", "limitations": "Benchmark and judge developed by Qwen; Qwen model rows are lab evaluations; other providers evaluated by benchmark authors. API snapshots/quality settings not fully specified." }, "reported_scale": "0–100", "benchmark_family": "qwen-image-bench-v1" }, { "id": "livecodebench-official-20240101-20241101", "name": "LiveCodeBench official (Jan 1–Nov 1, 2024)", "kind": "acc", "category": "llm", "url": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "family": "livecodebench-codegeneration", "description": "Official website code-generation pass@1. Inclusive UTC filter 2024-01-01 00:00:00 through 2024-11-01 00:00:00; 387 identical problem IDs per selected model. Mean then one-decimal percentage rounding as upstream. No difficulty components or repaired/composite scores.", "window_start": "2024-01-01T00:00:00Z", "window_end": "2024-11-01T00:00:00Z", "window_end_inclusive": true, "problem_count": 387, "release_commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70" }, { "id": "livecodebench-official-20240801-20250501", "name": "LiveCodeBench official (Aug 1, 2024–May 1, 2025)", "kind": "acc", "category": "llm", "url": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "family": "livecodebench-codegeneration", "description": "Official website code-generation pass@1. Inclusive UTC filter 2024-08-01 00:00:00 through 2025-05-01 00:00:00; 454 identical problem IDs per selected model. Mean then one-decimal percentage rounding as upstream. No difficulty components or repaired/composite scores.", "window_start": "2024-08-01T00:00:00Z", "window_end": "2025-05-01T00:00:00Z", "window_end_inclusive": true, "problem_count": 454, "release_commit": "f186ab5041d6b768733b8ae8145151c867942134" }, { "id": "gsm-plus", "name": "GSM-Plus", "kind": "acc", "category": "llm", "url": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B/blob/a07cc9a04f16550a088caea529712d1d335b0ac1/README.md", "description": "Robustness-oriented GSM-Plus math accuracy; distinct from GSM8K." }, { "id": "gsm8k-loose-substring", "name": "GSM8K loose substring accuracy", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2511.23404v1#A3", "description": "Liquid internal harness: correct answer as substring after extraction, rather than exact-match accuracy." }, { "id": "hmmt-feb-2026", "name": "HMMT February 2026", "kind": "acc", "category": "llm", "url": "https://huggingface.co/IFM/K2-Horizon-7B/blob/036114ce8d46c32b24c15423211069abb9c5d25e/README.md", "description": "February 2026 HMMT math contest." }, { "id": "ifeval-mean-four", "name": "IFEval mean of four measures", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2511.23404v1#A3", "description": "Arithmetic mean of strict/loose prompt-level and instruction-level accuracies. Distinct from any individual IFEval measure." }, { "id": "mgsm-liquid-six-language-loose", "name": "MGSM six languages, loose (Liquid)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2511.23404v1#A3", "description": "Mean loose accuracy over Spanish, French, German, Chinese, Japanese, Russian; distinct from full MGSM exact match." }, { "id": "mmmlu-liquid-seven-language", "name": "MMMLU seven languages (Liquid)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2511.23404v1#A3", "description": "OpenAI/MMMLU mean of AR_XY, DE_DE, ES_LA, FR_FR, JA_JP, KO_KR, ZH_CN; zero-shot without chat template.", "chance": 0.25 }, { "id": "multi-if-three-turn-average", "name": "Multi-IF three-turn average", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2511.23404v1#A3", "description": "Official Multi-IF reference implementation, reported mean accuracy over all3 turns; strict/loose aggregation not stated." }, { "id": "scicode-unspecified", "name": "SciCode (level unspecified)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/IFM/K2-Horizon-7B/blob/036114ce8d46c32b24c15423211069abb9c5d25e/README.md", "description": "Source labels only SciCode; main-vs-subtask is not inferred." }, { "id": "swe-bench-pro-split-unspecified", "name": "SWE-bench Pro (split unspecified)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/Qwen/Qwen3-Coder-Next/blob/a7fbcb5c0e12d62a448eaa0e260346bf5dcc0feb/README.md", "description": "Qwen release chart labels SWE-Bench Pro with SWE-Agent but does not identify public/private or a benchmark snapshot. Keep separate until split is established." }, { "id": "tau2-retail", "name": "τ²-bench Retail", "kind": "acc", "category": "llm", "url": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md", "description": "Retail domain pass rate; only used when no aggregate is provided." }, { "id": "tau2-telecom", "name": "τ²-bench Telecom", "kind": "acc", "category": "llm", "url": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md", "description": "Telecom domain pass rate; only used when no aggregate is provided." }, { "id": "terminal-bench-2.1", "name": "Terminal-Bench 2.1", "kind": "acc", "category": "llm", "url": "https://huggingface.co/IFM/K2-Horizon-7B/blob/036114ce8d46c32b24c15423211069abb9c5d25e/README.md", "description": "Terminal-Bench version 2.1; distinct from version 2.0." }, { "id": "amharic-addis-primary87-cer", "name": "Amharic TTS · 87-prompt CER (Chirp 3 judge)", "kind": "err", "category": "tts", "url": "https://huggingface.co/datasets/addisai/amharic-tts-benchmark/blob/a60556cd62dd2fbfce83a09c11b088e1e027ce64/data/summary.json", "metric": "character_error_rate", "setup": "2026-08-12. 100 fixed-seed test prompts: FLEURS34, Waxal33, Horn-ASR33; 87 without digits in primary table. Generated audio judged by Google Chirp3 after 16kHz mono conversion. NFC, punctuation removal, lowercasing, Ethiopic homophone folding, whitespace collapse; CER removes spaces. Micro sum of character edits/reference characters (8152). No audio imported.", "sample_count": 87, "score_unit": "error ratio; can exceed 1.0", "caveats": [ "Single run; model aliases can change after the run.", "OpenAI and ElevenLabs rows are unsupported-language conditions, not general TTS-quality claims.", "Google Gemini is judged with Google Chirp3; this is not a vendor-independent recognizer for that row.", "WER retained as companion metadata, not another ranked result; full100 aggregate and slices excluded.", "Raw error ratios above 1.0 are preserved. The score fit uses a negative-log error transform, not 1-error." ] }, { "id": "vibevoice-mit-figure-preference-mos", "name": "VibeVoice podcast · preference MOS (MIT figure)", "category": "tts", "kind": "acc", "metric": "mean_opinion_score", "url": "https://github.com/microsoft/VibeVoice/blob/1541f590c7099820f10ea012f48d2399282df69f/Figures/VibeVoice-TTS-results.jpg", "setup": "Long conversational/podcast speech; original report describes eight transcripts, about one hour total, 24 human annotators, six evaluated systems. Speech-reference conditioning except Gemini default male/female voices. The published conference paper Appendix H confirms preference rubric 1–5. Numerical values here are the separate, pinned repository figure snapshot; minor figure/table differences remain explicitly disclosed.", "scale_transform": "score = (raw MOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "MOS / 5; never display normalized score as recognition accuracy", "sample_count": 8, "annotator_count": 24, "setup_sources": [ "https://arxiv.org/html/2508.19205", "https://openreview.net/pdf?id=FihSkzyxdv" ], "caveats": [ "No explicit generation seed/immutable API build or figure evaluation date. Result date is snapshot review date.", "The three selected models overlap existing seed via Gemini 2.5 Pro Preview TTS.", "Eleven v3 Alpha is a distinct model ID from the production ElevenLabs v3.", "Only preference imported; no average/realism/richness duplication." ] }, { "id": "vapi-humanness-bt-20260925", "name": "Vapi Humanness · Bradley–Terry rating, 2026-09-25 snapshot", "category": "tts", "kind": "elo", "metric": "bradley_terry_rating", "url": "https://humannessindex.vapi.ai/api/models", "setup": "Blind listener preference for which speech sounds more human. Same fixed customer-support utterance and same cloned source voice on both sides; ties allowed. 20 fixed utterances with disfluencies; four licensed source voices (Clara, Emma, Godfrey, Nelliot), with pairing restricted to the voices available for both models. Frozen pre-generated clips. No audio imported. Vapi operates the evaluation and supplies no competing TTS model. Current API elo field is a Bradley–Terry fit on an Elo-like scale: live pairwise events plus original prototype aggregate counts as center-strength anchor games, prior=1. Ratings periodically recomputed and hourly cached; counts overlaid live. Exact ratings fit timestamp not returned.", "score_unit": "published rating points; higher is better", "prompt_count": 20, "setup_sources": [ "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/docs/METHODOLOGY.md", "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/arena.ts", "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/app/api/models/route.ts" ], "caveats": [ "Snapshot/export date is not an invented synthesis date or immutable backend version.", "Current BT and historical Elo are separate benchmarks and source groups; their numeric scales must not be merged.", "No human-normalized 0–100 composite or duplicate win-rate metric imported.", "Vote count is exposure count, not distinct listeners. API uncertainty is a rough 160/sqrt(n) estimate, not a confidence interval; omitted.", "For current ratings, live counts can be newer than the cached fit; they are retained as snapshot metadata, not exact fit sample size." ] }, { "id": "vapi-humanness-export-20260610", "name": "Vapi Humanness · historical Elo, 2026-06-10", "category": "tts", "kind": "elo", "metric": "elo_rating", "url": "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/seed-standings.json", "setup": "Blind listener preference for which speech sounds more human. Same fixed customer-support utterance and same cloned source voice on both sides; ties allowed. 20 fixed utterances with disfluencies; four licensed source voices (Clara, Emma, Godfrey, Nelliot), with pairing restricted to the voices available for both models. Frozen pre-generated clips. No audio imported. Vapi operates the evaluation and supplies no competing TTS model. Historical published Elo from the original prototype production export; only retired Turbo v2.5 retained to avoid duplicate current/historical rows.", "score_unit": "published rating points; higher is better", "prompt_count": 20, "setup_sources": [ "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/docs/METHODOLOGY.md", "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/src/server/arena.ts", "https://github.com/VapiAI/humanness-index/blob/e2b8f8653ed491a9f6038de98d6ef999c2729743/app/api/models/route.ts" ], "caveats": [ "Snapshot/export date is not an invented synthesis date or immutable backend version.", "Current BT and historical Elo are separate benchmarks and source groups; their numeric scales must not be merged.", "No human-normalized 0–100 composite or duplicate win-rate metric imported.", "Vote count is exposure count, not distinct listeners. API uncertainty is a rough 160/sqrt(n) estimate, not a confidence interval; omitted.", "For current ratings, live counts can be newer than the cached fit; they are retained as snapshot metadata, not exact fit sample size." ] }, { "id": "granite41-lscln-punctuation-error", "name": "Granite Speech · LScln punctuation error", "category": "stt", "kind": "err", "metric": "punctuation_error_rate", "url": "https://huggingface.co/ibm-granite/granite-speech-4.1-2b/blob/de575db64086f84fdc79da4932d1076e965bc546/README.md", "setup": "IBM own punctuation/truecasing evaluation of the base Granite Speech 4.1 2B model. Exact source test-set label LScln. PER follows LibriSpeech-PC, counting insertion, deletion and substitution of periods, commas and question marks. Cap-F1 is computed on Levenshtein-aligned matching word pairs and retained as a companion metric only.", "score_unit": "punctuation error ratio; not word error rate", "setup_sources": [ "https://huggingface.co/ibm-granite/granite-speech-4.1-2b/blob/de575db64086f84fdc79da4932d1076e965bc546/README.md", "https://arxiv.org/abs/2310.02943" ], "caveats": [ "Table does not provide sample counts, full decoding configuration or evaluation checkpoint commit. Snapshot revision pins evidence, not the actual run weights. Usage-example parameters are not treated as evaluation parameters.", "One measured model is useful as a raw record; insufficient overlap for a calibrated multi-model comparison.", "No WER guessed from graphs and no Open ASR leaderboard data copied.", "Other punctuation corpus cells remain in extraction evidence, avoiding a nine-benchmark expansion for one model." ] }, { "id": "vbench-t2v-subject-consistency-amd", "name": "VBench T2V — Subject consistency (AMD sample sets)", "category": "video", "kind": "acc", "chance": 0, "url": "https://github.com/Vchitect/VBench", "task": "text-to-video", "benchmark_family": "vbench", "reported_scale": "0–1", "setup": { "metric": "Subject consistency dimension only", "version": "VBench 1.0", "evaluation_scope": "AMD’s multiple generated-video sets; exact prompt list, sample count and generation settings not disclosed.", "comparability": "One of five correlated dimensions from the same AMD experiment. Not the official VBench total and not a complete standard leaderboard evaluation. Keep this scoped track separate from full-benchmark dimensions and totals.", "source_protocol": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#wan21-t2v-14b-benchmark" } }, { "id": "vbench-t2v-background-consistency-amd", "name": "VBench T2V — Background consistency (AMD sample sets)", "category": "video", "kind": "acc", "chance": 0, "url": "https://github.com/Vchitect/VBench", "task": "text-to-video", "benchmark_family": "vbench", "reported_scale": "0–1", "setup": { "metric": "Background consistency dimension only", "version": "VBench 1.0", "evaluation_scope": "AMD’s multiple generated-video sets; exact prompt list, sample count and generation settings not disclosed.", "comparability": "One of five correlated dimensions from the same AMD experiment. Not the official VBench total and not a complete standard leaderboard evaluation. Keep this scoped track separate from full-benchmark dimensions and totals.", "source_protocol": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#wan21-t2v-14b-benchmark" } }, { "id": "vbench-t2v-motion-smoothness-amd", "name": "VBench T2V — Motion smoothness (AMD sample sets)", "category": "video", "kind": "acc", "chance": 0, "url": "https://github.com/Vchitect/VBench", "task": "text-to-video", "benchmark_family": "vbench", "reported_scale": "0–1", "setup": { "metric": "Motion smoothness dimension only", "version": "VBench 1.0", "evaluation_scope": "AMD’s multiple generated-video sets; exact prompt list, sample count and generation settings not disclosed.", "comparability": "One of five correlated dimensions from the same AMD experiment. Not the official VBench total and not a complete standard leaderboard evaluation. Keep this scoped track separate from full-benchmark dimensions and totals.", "source_protocol": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#wan21-t2v-14b-benchmark" } }, { "id": "vbench-t2v-aesthetic-quality-amd", "name": "VBench T2V — Aesthetic quality (AMD sample sets)", "category": "video", "kind": "acc", "chance": 0, "url": "https://github.com/Vchitect/VBench", "task": "text-to-video", "benchmark_family": "vbench", "reported_scale": "0–1", "setup": { "metric": "Aesthetic quality dimension only", "version": "VBench 1.0", "evaluation_scope": "AMD’s multiple generated-video sets; exact prompt list, sample count and generation settings not disclosed.", "comparability": "One of five correlated dimensions from the same AMD experiment. Not the official VBench total and not a complete standard leaderboard evaluation. Keep this scoped track separate from full-benchmark dimensions and totals.", "source_protocol": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#wan21-t2v-14b-benchmark" } }, { "id": "vbench-t2v-imaging-quality-amd", "name": "VBench T2V — Imaging quality (AMD sample sets)", "category": "video", "kind": "acc", "chance": 0, "url": "https://github.com/Vchitect/VBench", "task": "text-to-video", "benchmark_family": "vbench", "reported_scale": "0–1", "setup": { "metric": "Imaging quality dimension only", "version": "VBench 1.0", "evaluation_scope": "AMD’s multiple generated-video sets; exact prompt list, sample count and generation settings not disclosed.", "comparability": "One of five correlated dimensions from the same AMD experiment. Not the official VBench total and not a complete standard leaderboard evaluation. Keep this scoped track separate from full-benchmark dimensions and totals.", "source_protocol": "https://github.com/ROCm/rocm-blogs/blob/b3bf907bd73c596c49c77b1d178c7c41bcd2713c/blogs/artificial-intelligence/mxfp-t2i-t2v/README.md#wan21-t2v-14b-benchmark" } }, { "id": "vbench-t2v-total", "name": "VBench (T2V total)", "category": "video", "kind": "acc", "chance": 0, "url": "https://github.com/Vchitect/VBench", "task": "text-to-video", "benchmark_family": "vbench", "reported_scale": "0–100", "setup": { "metric": "Published VBench overall total, divided by 100", "version": "VBench 1.0 T2V", "evaluation_scope": "Original full-set prompts in Jenga v2; baseline configuration and prompt format retained per row.", "comparability": "Not VBench-I2V, VBench 2.0, a dimension score, or an ad hoc subset average.", "source_protocol": "https://arxiv.org/pdf/2505.16864v2#page=7" } }, { "id": "vbench-t2v-total-mobilewan-480p", "name": "VBench T2V total (MobileWan 480p ablation)", "category": "video", "kind": "acc", "chance": 0, "url": "https://github.com/Vchitect/VBench", "task": "text-to-video", "benchmark_family": "vbench", "reported_scale": "0–100", "setup": { "metric": "Published VBench overall total, divided by 100", "version": "VBench T2V as reported in MobileWan v1 Table 10", "evaluation_scope": "Original Wan2.2-TI2V-5B forced to 81×480×832 at 16 FPS before fine-tuning; authors note 480p is not natively supported.", "comparability": "Separate non-native-resolution ablation. Do not treat it as native-resolution Wan2.2 performance or a MobileWan fine-tuned checkpoint. Prompt set completeness, seeds, sampler and NFE are not specified for this table.", "source_protocol": "https://arxiv.org/html/2607.06173v1#A1.SS4" } }, { "id": "target-bench-v2-vggt-wo-i2v", "name": "Target-Bench v2 (VGGT planning score, I2V)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2511.17792v2#S3", "task": "image-to-video", "benchmark_family": "target-bench", "reported_scale": "0–1", "setup": { "metric": "Published weighted overall (WO) score; higher is better.", "version": "arXiv 2511.17792v2 (2026-04-15), Table 3", "evaluation_scope": "125 held-out robot scenarios, curated from 450 scenarios in 47 semantic target categories; remaining 325 only used for training. First image and semantic text condition; 8-second, 720p base-model video in Table 3. VGGT trajectory reconstruction and metric scale recovery.", "metric_formula": "WO = 0.075*exp(-ADE/1m) + 0.125*exp(-FDE/1m) + 0.125*(1-MR/100) + 0.675*SE*AC; copy the published overall, do not recalculate from rounded aggregate columns.", "comparability": "Version 2 metric and VGGT decoder only; no mixing with v1, SpaTracker, ViPE, later decoder changes, generic visual-quality scores, or T2V. The model is not fine-tuned.", "source_protocol": "https://arxiv.org/html/2511.17792v2#S3.SS2.SSSx2.Px7.p1" } }, { "id": "agieval-marco-unspecified", "name": "AGIEval (Marco suite unspecified)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2604.25578v1", "description": "Marco report names AGIEval accuracy but does not enumerate selected subtests; distinct from English-only nine-test aggregate." }, { "id": "browsecomp-ling-single-context", "name": "BrowseComp single-agent with context management", "kind": "acc", "category": "llm", "url": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md", "description": "Internal single-agent ReAct harness; summarize history at64K and resume; average pass@1. Separate from multi-agent82.0 score." }, { "id": "c-eval", "name": "C-Eval", "kind": "acc", "category": "llm", "url": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md", "description": "Chinese examination benchmark aggregate; split/scoring retained from source.", "chance": 0.25 }, { "id": "chinese-simpleqa", "name": "ChineseSimpleQA", "kind": "acc", "category": "llm", "url": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md", "description": "ChineseSimpleQA factual-answer accuracy." }, { "id": "cmmlu", "name": "CMMLU", "kind": "acc", "category": "llm", "url": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md", "description": "Chinese Massive Multitask Language Understanding aggregate.", "chance": 0.25 }, { "id": "codegolf-v2-2", "name": "Codegolf v2.2", "kind": "acc", "category": "llm", "url": "https://ai.google.dev/gemma/docs/gemma-3n/model_card", "description": "Codegolf v2.2 pass@1 as named by Google; no mapping to HumanEval or other coding suites." }, { "id": "global-mmlu", "name": "Global-MMLU", "kind": "acc", "category": "llm", "url": "https://ai.google.dev/gemma/docs/gemma-3n/model_card", "description": "Global-MMLU accuracy; not Global-MMLU-Lite. Language coverage/aggregation retained as stated by each source.", "chance": 0.25 }, { "id": "global-mmlu-marco", "name": "GlobalMMLU (Marco evaluation)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2604.25578v1", "description": "Single generated response per query; source’s multilingual aggregation, language subset not enumerated.", "chance": 0.25 }, { "id": "gpqa-diamond-google-relaxed", "name": "GPQA Diamond (Google RelaxedAccuracy/accuracy)", "kind": "acc", "category": "llm", "url": "https://ai.google.dev/gemma/docs/gemma-3n/model_card", "description": "Google card reports Metric=RelaxedAccuracy/accuracy without resolving which scorer produced this cell; kept separate from other GPQA Diamond scoring.", "chance": 0.25 }, { "id": "imo-answerbench", "name": "IMO-AnswerBench", "kind": "acc", "category": "llm", "url": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md", "description": "IMO-AnswerBench answer accuracy; no majority-vote aggregation assumed." }, { "id": "kcl-essay-vanilla", "name": "KCL-Essay vanilla", "kind": "acc", "category": "llm", "url": "https://aclanthology.org/2026.eacl-short.17.pdf", "description": "169 legal essays; Gemini2.5Flash instance-level rubric judge; percent of2905 total rubric-weighted points, averaged over3 independent runs; no supporting precedents." }, { "id": "kcl-mcqa-vanilla", "name": "KCL-MCQA vanilla", "kind": "acc", "category": "llm", "url": "https://aclanthology.org/2026.eacl-short.17.pdf", "description": "283 five-option legal questions, no supporting precedents; percent accuracy averaged over3 independent runs.", "chance": 0.2 }, { "id": "livecodebench-202408-202505-version-unspecified", "name": "LiveCodeBench 2024-08–2025-05 (version unstated)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md", "description": "Source explicitly labels 2408–2505 window but not release version. Kept separate from a v6 claim." }, { "id": "livecodebench-v5-window-unspecified", "name": "LiveCodeBench v5 (window unspecified)", "kind": "acc", "category": "llm", "url": "https://ai.google.dev/gemma/docs/gemma-3n/model_card", "description": "Explicit v5 pass@1; date filter not specified, not merged with a different filtered v5 setup." }, { "id": "mbpp", "name": "MBPP", "kind": "acc", "category": "llm", "url": "https://ai.google.dev/gemma/docs/gemma-3n/model_card", "description": "Original MBPP pass@1; not MBPP+ expanded test suite." }, { "id": "mcp-atlas-v1-500", "name": "MCP-Atlas v1 public500", "kind": "acc", "category": "llm", "url": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md", "description": "500 public tasks; official v1 harness,20-turn limit; Gemini2.5Pro claim-coverage judge." }, { "id": "mgsm-marco", "name": "MGSM (Marco evaluation)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2604.25578v1", "description": "Multilingual GSM exact match, Avg@8; no assumption about language aggregation." }, { "id": "mmlu-prox-lite-marco", "name": "MMLU-ProX-Lite (Marco)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2604.25578v1", "description": "Multilingual MMLU-ProX-Lite accuracy; Avg@8; language aggregation not enumerated." }, { "id": "mmmlu-marco", "name": "MMMLU (Marco evaluation)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2604.25578v1", "description": "OpenAI/MMMLU single generated response per query; exact language subset not enumerated, separate from seven-language or known full aggregation.", "chance": 0.25 }, { "id": "multi-if-unspecified", "name": "Multi-IF (aggregation unspecified)", "kind": "acc", "category": "llm", "url": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md", "description": "Published Multi-IF score; source does not resolve turn/strictness aggregation." }, { "id": "musr", "name": "MuSR", "kind": "acc", "category": "llm", "url": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md", "description": "Multistep soft-reasoning aggregate; no task components also imported." }, { "id": "polymath-marco", "name": "PolyMath (Marco evaluation)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2604.25578v1", "description": "PolyMath exact-match Avg@8 as reported; language/difficulty aggregation not further specified." }, { "id": "skillsbench-87", "name": "SkillsBench 87-task subset", "kind": "acc", "category": "llm", "url": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md", "description": "Kilo-code harness;87 tasks excluding external-API-dependent tasks; mean over3runs." }, { "id": "sysbench-unspecified", "name": "SysBench (aggregation unspecified)", "kind": "acc", "category": "llm", "url": "https://github.com/PaddlePaddle/ERNIE/blob/790a50b045d1aca2753d5395d8bec0806b2e6925/README.md", "description": "Published SysBench system-instruction benchmark score; exact aggregation/snapshot unspecified." }, { "id": "tau2-airline", "name": "τ²-bench Airline", "kind": "acc", "category": "llm", "url": "https://huggingface.co/ServiceNow-AI/Apriel-1.6-15b-Thinker/blob/5810d5f1723032300131e677450d7f95a15a05e7/README.md", "description": "Airline-domain task success; no composite or Telecom value also imported from this source." }, { "id": "thunder-konubench-symbol-zero-shot", "name": "Thunder-KoNUBench symbol 0-shot", "kind": "acc", "category": "llm", "url": "https://aclanthology.org/2026.findings-acl.324.pdf", "description": "Published ACL2026 Table16 zero-shot accuracy: choose A/B/C/D symbol with highest log-likelihood. No character-normalized cloze scores or fine-tuned checkpoints mixed in.", "chance": 0.25 }, { "id": "widesearch-corrected", "name": "WideSearch corrected dataset", "kind": "acc", "category": "llm", "url": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/ef06d91fe382109ae82647da88ff99b0f11745b0/README.md", "description": "Corrected WideSearch with official prompt; GPT4.1 judge; own internal ReAct search harness, average pass@1." }, { "id": "cv3-eval-ml9-macro-err", "name": "CV3-Eval multilingual voice clone · 9-language mean error", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2607.23938v1", "metric": "mixed_word_character_error_rate", "setup": "CV3-Eval Multilingual Voice Cloning subset (CosyVoice 3 release), zero-shot cloning from the CV3 prompt. Unweighted mean over zh, en, ja, ko, de, es, fr, it, ru; CER for zh/ja/ko, WER otherwise; original CV3/Seed scoring stack (Paraformer for Chinese, Whisper-large-v3 otherwise). Each evaluator is its own series; per-language cells are provenance, not extra results.", "score_unit": "error ratio (source % / 100)", "setup_sources": [ "https://arxiv.org/html/2607.23938v1", "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md" ], "caveats": [ "Mixed CER/WER mean; not a homogeneous WER.", "API models lack immutable build identifiers.", "Boson's macro (sglang-omni cookbook) publishes only the mean, not per-language cells." ] }, { "id": "cv3-eval-ext7-macro-err", "name": "CV3-Eval protocol, 7 added languages · mean WER (Qwen-Audio 3.0 extension)", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2607.23938v1", "metric": "word_error_rate", "setup": "Qwen-Audio-3.0-TTS report Table 4 extension of the CV3-Eval multilingual voice-cloning protocol to ar, id, pt, th, vi, ms, tl. Unweighted mean over the seven languages; only systems with all seven values.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Test items for the added languages are the report authors' extension, not part of the released CV3-Eval.", "Evaluated by a competitor of the API vendors." ] }, { "id": "cv3-eval-hard-zh-cer", "name": "CV3-Eval hard-zh · CER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2607.23938v1", "metric": "character_error_rate", "setup": "CV3-Eval hard-zh subset (difficult Chinese texts), zero-shot voice cloning, Paraformer CER.", "score_unit": "error ratio (source % / 100)" }, { "id": "cv3-eval-hard-en-wer", "name": "CV3-Eval hard-en · WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2607.23938v1", "metric": "word_error_rate", "setup": "CV3-Eval hard-en subset (difficult English texts), zero-shot voice cloning, Whisper-large-v3 WER.", "score_unit": "error ratio (source % / 100)" }, { "id": "ztts1-clean-ml9-wer", "name": "ZTTS1-Eval Clean (FLEURS-R) · 9-language mean WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2606.24320v2", "metric": "word_error_rate", "setup": "ZTTS1-Eval Clean: 500 FLEURS-R read-speech utterances per language; zero-shot cloning where supported, preset voices otherwise. Qwen3-ASR scorer. Unweighted mean over en, zh, de, es, fr, it, ja, ko, ru (hard_en/hard_zh excluded).", "score_unit": "error ratio (source % / 100)", "caveats": [ "Share-alike source (CC BY-SA 4.0).", "Preset-voice systems (ElevenLabs v3, Gemini) do not clone the prompt speaker.", "Scored by the ZONOS2 developer." ] }, { "id": "ztts1-clean-ml9-utmos", "name": "ZTTS1-Eval Clean (FLEURS-R) · 9-language mean MSR-UTMOS", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2606.24320v2", "metric": "predicted_mos_utmos", "setup": "Same generations as ztts1-clean-ml9-wer; MSR-UTMOS (utmos22_strong) automatic naturalness prediction, unweighted mean over the nine languages.", "scale_transform": "score = (raw UTMOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "predicted MOS / 5; not human MOS, not accuracy", "caveats": [ "Automatic MOS predictor trained mainly on English; multilingual scores are indicative.", "Share-alike source (CC BY-SA 4.0)." ] }, { "id": "ztts1-itw-10lang-wer", "name": "ZTTS1-Eval in-the-wild · 10-language mean WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2606.24320v2", "metric": "word_error_rate", "setup": "ZTTS1-Eval in-the-wild set: spontaneous VoxBlink2 prompts (1,618 utterances, 17 languages). Qwen3-ASR scorer. Unweighted mean over the 10 languages every compared system supports: en, zh, de, es, fr, it, ja, ko, pt, ru. No prompt audio imported.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Share-alike source (CC BY-SA 4.0).", "Preset-voice systems do not clone the noisy prompt speaker; they read with a clean built-in voice." ] }, { "id": "ztts1-itw-10lang-utmos", "name": "ZTTS1-Eval in-the-wild · 10-language mean MSR-UTMOS", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2606.24320v2", "metric": "predicted_mos_utmos", "setup": "Same generations as ztts1-itw-10lang-wer; MSR-UTMOS mean over the same 10 languages.", "scale_transform": "score = (raw UTMOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "predicted MOS / 5; not human MOS, not accuracy", "caveats": [ "Automatic MOS predictor.", "Share-alike source (CC BY-SA 4.0)." ] }, { "id": "rw-voice-eq-tts-acting", "name": "RW-Voice-EQ TTS · Acting / Role-Fit (human)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2607.14846v1", "metric": "human_rating_factor_score", "setup": "RW-Voice-EQ Bench TTS (Hume AI): 31 system configurations, provider default voice/settings, 3 generations per model-prompt, each clip rated by 3 blind human raters (Hume Study Runner; US/UK/CA native English, language-matched for code-switching). Per-eval composites z-scored, mixed-effects rater-controlled provider means; the dimension score is the equal-weighted mean of its evals, printed on the rating scale. Audiobook narration, sports commentary, stand-up, humorous delivery, casual reading, vocal bursts (66 prompts): naturalness, prosody, role fit.", "scale_transform": "score = (raw - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "rating / 5; not accuracy", "setup_sources": [ "https://arxiv.org/html/2607.14846v1", "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg" ], "caveats": [ "Only the top five systems per dimension are printed (truncated series).", "Model-level mean ± SD printed; underlying per-eval data are in a private dataset and were not used." ] }, { "id": "rw-voice-eq-tts-expressiveness", "name": "RW-Voice-EQ TTS · Expressiveness (human)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2607.14846v1", "metric": "human_rating_factor_score", "setup": "RW-Voice-EQ Bench TTS (Hume AI): 31 system configurations, provider default voice/settings, 3 generations per model-prompt, each clip rated by 3 blind human raters (Hume Study Runner; US/UK/CA native English, language-matched for code-switching). Per-eval composites z-scored, mixed-effects rater-controlled provider means; the dimension score is the equal-weighted mean of its evals, printed on the rating scale. Inferred/instructed emotion, pragmatic subtext, sarcasm, expressive constraints, mid-utterance transitions (120 prompts).", "scale_transform": "score = (raw - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "rating / 5; not accuracy", "setup_sources": [ "https://arxiv.org/html/2607.14846v1", "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg" ], "caveats": [ "Only the top five systems per dimension are printed (truncated series).", "Model-level mean ± SD printed; underlying per-eval data are in a private dataset and were not used." ] }, { "id": "rw-voice-eq-tts-voice-identity", "name": "RW-Voice-EQ TTS · Voice Identity (human)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2607.14846v1", "metric": "human_rating_factor_score", "setup": "RW-Voice-EQ Bench TTS (Hume AI): 31 system configurations, provider default voice/settings, 3 generations per model-prompt, each clip rated by 3 blind human raters (Hume Study Runner; US/UK/CA native English, language-matched for code-switching). Per-eval composites z-scored, mixed-effects rater-controlled provider means; the dimension score is the equal-weighted mean of its evals, printed on the rating scale. Speaker consistency under emotional stress, physical states, prosodic extremes, register shifts (96 prompts); same-speaker confidence.", "scale_transform": "score = (raw - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "rating / 5; not accuracy", "setup_sources": [ "https://arxiv.org/html/2607.14846v1", "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg" ], "caveats": [ "Only the top five systems per dimension are printed (truncated series).", "Model-level mean ± SD printed; underlying per-eval data are in a private dataset and were not used." ] }, { "id": "rw-voice-eq-tts-language-switching", "name": "RW-Voice-EQ TTS · Language Code-Switching (human)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2607.14846v1", "metric": "human_rating_factor_score", "setup": "RW-Voice-EQ Bench TTS (Hume AI): 31 system configurations, provider default voice/settings, 3 generations per model-prompt, each clip rated by 3 blind human raters (Hume Study Runner; US/UK/CA native English, language-matched for code-switching). Per-eval composites z-scored, mixed-effects rater-controlled provider means; the dimension score is the equal-weighted mean of its evals, printed on the rating scale. Within-utterance code-switching DE/ES/FR/IT/JA and heteronyms (70 prompts); native-accent perception and foreign-phrase comprehension.", "scale_transform": "score = (raw - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "rating / 5; not accuracy", "setup_sources": [ "https://arxiv.org/html/2607.14846v1", "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg" ], "caveats": [ "Only the top five systems per dimension are printed (truncated series).", "Model-level mean ± SD printed; underlying per-eval data are in a private dataset and were not used.", "Table 1 describes this dimension's questions as yes/no or % accuracy, but the printed factor scores lie on the same ~1-5 scale as the other dimensions; normalization assumes 1-5 bounds and the raw value is preserved." ] }, { "id": "rw-voice-eq-tts-reliability", "name": "RW-Voice-EQ TTS · Reliability (human)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2607.14846v1", "metric": "human_rating_factor_score", "setup": "RW-Voice-EQ Bench TTS (Hume AI): 31 system configurations, provider default voice/settings, 3 generations per model-prompt, each clip rated by 3 blind human raters (Hume Study Runner; US/UK/CA native English, language-matched for code-switching). Per-eval composites z-scored, mixed-effects rater-controlled provider means; the dimension score is the equal-weighted mean of its evals, printed on the rating scale. Pharmaceutical terms, alphanumeric strings, structured data reading/spelling (64 prompts); correct/clear read-back.", "scale_transform": "score = (raw - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "rating / 5; not accuracy", "setup_sources": [ "https://arxiv.org/html/2607.14846v1", "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg" ], "caveats": [ "Only the top five systems per dimension are printed (truncated series).", "Model-level mean ± SD printed; underlying per-eval data are in a private dataset and were not used.", "Table 1 describes this dimension's questions as yes/no or % accuracy, but the printed factor scores lie on the same ~1-5 scale as the other dimensions; normalization assumes 1-5 bounds and the raw value is preserved." ] }, { "id": "rw-voice-eq-tts-longform-stability", "name": "RW-Voice-EQ TTS · Long-Form Stability (human)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2607.14846v1", "metric": "human_rating_factor_score", "setup": "RW-Voice-EQ Bench TTS (Hume AI): 31 system configurations, provider default voice/settings, 3 generations per model-prompt, each clip rated by 3 blind human raters (Hume Study Runner; US/UK/CA native English, language-matched for code-switching). Per-eval composites z-scored, mixed-effects rater-controlled provider means; the dimension score is the equal-weighted mean of its evals, printed on the rating scale. Long-form web reading and identity persistence/transitions (80 prompts); mean of the two long-form evaluation ranges.", "scale_transform": "score = (raw - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "rating / 5; not accuracy", "setup_sources": [ "https://arxiv.org/html/2607.14846v1", "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg" ], "caveats": [ "Only the top five systems per dimension are printed (truncated series).", "Model-level mean ± SD printed; underlying per-eval data are in a private dataset and were not used." ] }, { "id": "rw-voice-eq-tts-acoustic-quality", "name": "RW-Voice-EQ TTS · Acoustic Quality (human)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2607.14846v1", "metric": "human_rating_factor_score", "setup": "RW-Voice-EQ Bench TTS (Hume AI): 31 system configurations, provider default voice/settings, 3 generations per model-prompt, each clip rated by 3 blind human raters (Hume Study Runner; US/UK/CA native English, language-matched for code-switching). Per-eval composites z-scored, mixed-effects rater-controlled provider means; the dimension score is the equal-weighted mean of its evals, printed on the rating scale. Artifact/fidelity ratings collected within the expression evaluations (reverse-coded so higher is better).", "scale_transform": "score = (raw - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "rating / 5; not accuracy", "setup_sources": [ "https://arxiv.org/html/2607.14846v1", "https://arxiv.org/html/2607.14846v1/TTS_Top5_Membership_Matrix.svg" ], "caveats": [ "Only the top five systems per dimension are printed (truncated series).", "Model-level mean ± SD printed; underlying per-eval data are in a private dataset and were not used." ] }, { "id": "rw-voice-eq-asr-avg4-wer", "name": "RW-Voice-EQ ASR · 4 real-world English sets, mean WER", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2607.14846v1", "metric": "word_error_rate", "setup": "Hume AI human-verified private English sets: accented (230 clips, 23 accents), emotional (620 clips), background noise/music (460 clips), conversational Hume-DaiKon (14,142 segments). Corpus WER per set with Open ASR Leaderboard normalization; unweighted mean of the four. Only the top ten of 40 systems are printed (Figure 8).", "score_unit": "error ratio (source % / 100)", "setup_sources": [ "https://arxiv.org/html/2607.14846v1", "https://arxiv.org/html/2607.14846v1/avg_wer_bar_top10.svg" ], "caveats": [ "Private audio; not reproducible externally.", "Truncated to the ten best systems.", "Vendor API builds not pinned." ] }, { "id": "seed-tts-en-zh-macro-err", "name": "Seed-TTS-Eval · test-en/test-zh macro mean error", "category": "tts", "kind": "err", "url": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md", "metric": "mixed_word_character_error_rate", "setup": "Seed-TTS-Eval zero-shot voice cloning; unweighted mean of test-en WER (Whisper-large-v3) and test-zh CER (Paraformer) with the original seed-tts-eval normalization, as published by the evaluator (only the macro is printed). Distinct from the separate en/zh benchmarks.", "score_unit": "error ratio (source x100 / 100)", "caveats": [ "Only the two-language macro is published; components unknown." ] }, { "id": "minimax-ml23-macro-err", "name": "MiniMax multilingual test set · 23-language mean error", "category": "tts", "kind": "err", "url": "https://github.com/sgl-project/sglang-omni/blob/0aafd76b4ef488f1c01987cc856cea77af129d61/docs/cookbook/higgs_tts.md", "metric": "mixed_word_character_error_rate", "setup": "MiniMax-Speech multilingual voice-cloning test set (two Common Voice prompts per language), 23 languages, original Seed-TTS-style metrics (CER for CJK/Thai-type languages, WER otherwise), macro mean as published by Boson AI. Different language set from voxtral-tts-minimax9-wer.", "score_unit": "error ratio (source x100 / 100)", "caveats": [ "Only the macro is published in the pinned table.", "Includes languages some systems do not support." ] }, { "id": "lemas-eval-9lang-wer", "name": "LEMAS multilingual TTS eval · 9-language mean WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2601.04233v1", "metric": "word_error_rate", "setup": "LEMAS-Dataset evaluation set, 500 utterances per language (3-15 s, alignment score > 0.9), zero-shot cloning; Paraformer-zh for Chinese, Whisper-large-v3 otherwise (Seed-TTS protocol). Mean over de,en,es,fr,id,it,pt,ru,zh as printed for OpenAudio-S1-Mini (vi excluded, no value).", "score_unit": "error ratio (source % / 100)", "caveats": [ "Share-alike source (CC BY-SA 4.0).", "Only one catalogued system; raw evidence, not a calibrated comparison." ] }, { "id": "emergent-tts-v1-paper-wvmos", "name": "EmergentTTS · v1 paper automatic MOS (WV-MOS)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2505.23009v1", "metric": "predicted_mean_opinion_score", "setup": "EmergentTTS-Eval v1 paper Table 1 (May 2025): 1,645 English test cases in six categories. MOS predicted per clip by a fine-tuned wav2vec2.0 model (WV-MOS; repository quality_assessment/wv_mos.py adopted from AndreevP/wvmos) and averaged; automatic proxy, not human MOS. Same generated audio as the README six-category leaderboard (per-category WERs identical). Voice and strong-prompting (†) recorded per result.", "raw_scale_min": 1, "raw_scale_max": 5, "scale_transform": "score = (raw MOS - 1) / 4", "display_unit": "predicted MOS / 5; never display normalized score as recognition accuracy", "sample_count": 1645, "setup_sources": [ "https://arxiv.org/html/2505.23009v1", "https://github.com/boson-ai/EmergentTTS-Eval-public/blob/fe02c5d27f2d87751a66e32631aaa73a92d9cbbb/quality_assessment/wv_mos.py" ], "caveats": [ "WV-MOS is an automatic naturalness predictor with no hard upper bound; all selected values lie within 3.0-4.4.", "Same audio as the README WER/preference rows: a distinct metric, not an independent study.", "Paper table predates the README-only additions (Gemini, MiniMax, Kokoro, Chatterbox, Higgs); only paper rows imported.", "Voices differ by system (Alloy, Thalia-en, Brian, Tara, exampleaudio; Hume and Sesame use per-utterance/default voices)." ] }, { "id": "ztts1-fleursr-en-ttsds2-prosody", "name": "ZTTS1-Eval · FLEURS-R English TTSDS2 prosody", "category": "tts", "kind": "acc", "url": "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/RESULTS_CURRENT.md", "metric": "distribution_similarity_score", "setup": "Zyphra ZTTS1-Eval zero-shot set built from google/fleurs-r (English). TTSDS2 (ttsds v2.1.1) prosody factor: mean of MPM, Pitch, Hubert-Token-SR and Allosaurus-SR distributional benchmarks, each normalized by the real-to-noise distance, 0-100 divided by 100. Cloning models condition on the FLEURS-R prompt speaker; Gemini uses deterministically rotated prebuilt voices (no cloning).", "score_unit": "source 0-100 divided by 100", "setup_sources": [ "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/RESULTS_CURRENT.md", "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/run_generate_omni.sh", "https://github.com/Zyphra/ZTTS1-Eval/blob/93660731c7b8d5e70935975fe7a6c7ed48451a7b/scripts/generate_gemini.py" ], "caveats": [ "Distribution similarity to real speech, not recognition accuracy or MOS; distinct from TTSDS v1 four-factor default.", "Same Zyphra run family as the ZONOS2 report; its WER_CLIP/speaker-similarity/UTMOS tables are deliberately not imported here to avoid double counting.", "Zyphra evaluates its own ZONOS2 (lab) alongside competitors (independent).", "Preset-voice Gemini versus cloning models is a different conditioning regime.", "API build/weight revisions not pinned beyond model ids; generation seeds not reported." ] }, { "id": "granite5-report-oasr-public7-wer", "name": "Granite 5.0 report · Open ASR public 7-set mean WER", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2609.20104v1", "metric": "word_error_rate", "setup": "IBM's own evaluation in arXiv:2609.20104v1 Table 3 on the public components of the Open ASR leaderboard: AMI, Earnings22, Gigaspeech, LibriSpeech clean, LibriSpeech other, SPGISpeech, VoxPopuli (7 sets, TED-LIUM absent). AVG is the unweighted mean of the seven per-set WERs. Report text states Table 5/6 used the 20 May 2026 leaderboard version; Table 3 set version assumed the same public components. Greedy CTC decoding. Not the 8-set March-2026 Open ASR CSV mix (open-asr-wer) and not the leaderboard's private Voice Arena set.", "caveats": [ "Lab self-report; single-model series in this source.", "Table 3 row 'Robust fine tuning' is the final of three training stages described for the released model; the table does not print the Hugging Face revision.", "Do not merge with open-asr-wer: different set mix (no TED-LIUM) and different evaluator." ] }, { "id": "qwen-audio3-asr-panelb-zh4-cer", "name": "Qwen-Audio-3.0-ASR report · Chinese 4-set CER mean (own API runs)", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2609.07549v2", "metric": "character_error_rate", "setup": "arXiv:2609.07549v2 Table 1 Panel B: every system run by the Qwen-Audio ASR team through its own unified API-based evaluation pipeline on the same test sets. WeNetSpeech-net uses corrected reference transcripts (WenetSpeech issue #63). API build dates, request parameters and normalization details not published. Azure (unversioned) and Doubao-ASR (unversioned product label) columns not imported. Derived unweighted mean of AISHELL-1, AISHELL-2, FLEURS-zh and WeNetSpeech-net (test_net) CER.", "caveats": [ "Evaluator is the vendor of Qwen-Audio-3.0-ASR and Fun-ASR (by=lab for those two).", "Macro mean derived by this project; per-set values kept on each row.", "Commercial API versions change over time; results are a September-2026 report snapshot." ] }, { "id": "qwen-audio3-asr-panelb-en3-wer", "name": "Qwen-Audio-3.0-ASR report · English 3-set WER mean (own API runs)", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2609.07549v2", "metric": "word_error_rate", "setup": "arXiv:2609.07549v2 Table 1 Panel B: every system run by the Qwen-Audio ASR team through its own unified API-based evaluation pipeline on the same test sets. WeNetSpeech-net uses corrected reference transcripts (WenetSpeech issue #63). API build dates, request parameters and normalization details not published. Azure (unversioned) and Doubao-ASR (unversioned product label) columns not imported. Derived unweighted mean of FLEURS-en, LibriSpeech test-clean and LibriSpeech test-other WER.", "caveats": [ "Evaluator is the vendor of Qwen-Audio-3.0-ASR and Fun-ASR (by=lab for those two).", "Macro mean derived by this project; per-set values kept on each row." ] }, { "id": "verli-librispeech-clean-wer", "name": "verli STT benchmark · LibriSpeech test-clean WER", "category": "stt", "kind": "err", "url": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_librispeech.json", "metric": "word_error_rate", "setup": "verliapp/stt-benchmark: every engine gets the same files, scored by the same score_ood.py (OpenAI Whisper English normalizer, corpus WER = total edits / total reference words, 95% utterance bootstrap CI). Cloud APIs called 2026-07-16..18 through pre-recorded/batch endpoints with pinned model ids (cloud_adapters.py); on-device Parakeet via parakeet-mlx (mlx-community conversions). Engines scored only on clips they returned; coverage recorded. LibriSpeech test-clean, all 2,620 utterances, clean read speech.", "caveats": [ "Single run per engine; hosted model aliases can change after the run.", "AssemblyAI identity rests on the adapter default speech_models=['universal-3-5-pro'] at the results commit and the README label 'AssemblyAI Universal-3.5'; the results JSON records only the engine label.", "Vendor terms of service may restrict publishing benchmarks; that obligation falls on the benchmarker, not this reuse of MIT-licensed numbers." ] }, { "id": "verli-earnings22-wer", "name": "verli STT benchmark · Earnings-22 WER (2,741 utt.)", "category": "stt", "kind": "err", "url": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_earnings22.json", "metric": "word_error_rate", "setup": "verliapp/stt-benchmark: every engine gets the same files, scored by the same score_ood.py (OpenAI Whisper English normalizer, corpus WER = total edits / total reference words, 95% utterance bootstrap CI). Cloud APIs called 2026-07-16..18 through pre-recorded/batch endpoints with pinned model ids (cloud_adapters.py); on-device Parakeet via parakeet-mlx (mlx-community conversions). Engines scored only on clips they returned; coverage recorded. Earnings-22 test from the Open ASR Leaderboard ESB bundle, 2,741 utterances of earnings-call audio.", "caveats": [ "Single run per engine; hosted model aliases can change after the run.", "AssemblyAI identity rests on the adapter default speech_models=['universal-3-5-pro'] at the results commit and the README label 'AssemblyAI Universal-3.5'; the results JSON records only the engine label.", "Vendor terms of service may restrict publishing benchmarks; that obligation falls on the benchmarker, not this reuse of MIT-licensed numbers." ] }, { "id": "verli-spgispeech1200-wer", "name": "verli STT benchmark · SPGISpeech 1,200-clip sample WER", "category": "stt", "kind": "err", "url": "https://github.com/verliapp/stt-benchmark/blob/bfb51427ba12380ff1e3620a8333c10f18fa4280/results/results_spgispeech.json", "metric": "word_error_rate", "setup": "verliapp/stt-benchmark: every engine gets the same files, scored by the same score_ood.py (OpenAI Whisper English normalizer, corpus WER = total edits / total reference words, 95% utterance bootstrap CI). Cloud APIs called 2026-07-16..18 through pre-recorded/batch endpoints with pinned model ids (cloud_adapters.py); on-device Parakeet via parakeet-mlx (mlx-community conversions). Engines scored only on clips they returned; coverage recorded. SPGISpeech test, 1,200-clip sample (prep_ood_dataset.py --limit 1200), financial calls.", "caveats": [ "Single run per engine; hosted model aliases can change after the run.", "AssemblyAI identity rests on the adapter default speech_models=['universal-3-5-pro'] at the results commit and the README label 'AssemblyAI Universal-3.5'; the results JSON records only the engine label.", "Vendor terms of service may restrict publishing benchmarks; that obligation falls on the benchmarker, not this reuse of MIT-licensed numbers." ] }, { "id": "koedesk-fleurs200-9lang-headline-mean", "name": "koedesk FLEURS first-200 · 9-language headline error mean", "category": "stt", "kind": "err", "url": "https://huggingface.co/datasets/koedesk/stt-bench/blob/c97de119c0511900d559bde5024c2fdc31be0b2c/scores.json", "metric": "mixed_word_character_error_rate", "setup": "Preregistered koedesk STT Bench v1 Track A, run 2026-07-05: google/fleurs test split, first 200 utterances per language, audio peak-normalized to -6 dBFS 16-bit PCM for every engine, explicit target language on every call, no biasing. Preregistered normalization (Whisper normalizer for en/de/fr/es/pt, per-language rules otherwise); corpus-level rates; empty outputs scored as deletions. Headline metric per language: CER for ja and zh, WER for de, en, es, fr, ko, pt, ru. Derived unweighted mean over those nine languages, the set covered by every selected engine (Voxtral has no id/th/vi). id, th, vi excluded for all models.", "sample_count": 1800, "caveats": [ "Mixed WER/CER macro mean; do not label as homogeneous WER.", "FLEURS is public; engines may have trained on it (source's own caveat).", "Benchmark author uses ElevenLabs Scribe v2 in its product (disclosed conflict of interest).", "Derived nine-language mean by this project; per-language values kept on each row." ] }, { "id": "raven-de-read4-wer", "name": "raven.eval · German read/parliament 4-set WER mean (100 utt. each)", "category": "stt", "kind": "err", "url": "https://github.com/PhilflowIO/raven.eval/blob/f9f621e23b0e83e29aa18e8ffcd4656805c68145/BENCHMARKS.md", "metric": "word_error_rate", "setup": "raven.eval (commit f9f621e): first 100 rows each of avemio/ASR-GERMAN-MIXED-TEST, google/fleurs de_de test, facebook/multilingual_librispeech german test and facebook/voxpopuli de test at pinned HF revisions; flozi-strict normalization, per-utterance alignment (since 2026-09-19), corpus WER = total edits / total reference words; 95% utterance bootstrap (10,000). Derived unweighted mean of the four published 'WER strict %' values. Swiss-German dialect sets (spc-test, fhnw) excluded: translation-shaped, BLEU headline.", "sample_count": 400, "caveats": [ "Only 100 utterances per set; source says overlapping 95% intervals do not rank models.", "Hosted API run once (us-east-1, language=de, format=true) on 2026-09-18.", "Grok Voice Transcribe 1.0 was measured on the same samples but is not in the catalogue; not imported.", "Vendor terms may restrict publishing benchmarks; obligation on the benchmarker." ] }, { "id": "bangla-riverborn-1001-batch-cer", "name": "Bangla STT benchmark · BanSpeech 1,001-clip batch CER", "category": "stt", "kind": "err", "url": "https://github.com/riverbornai/bangla-speech-to-text-benchmark/blob/cc73eb073b065bc4c7301672a2013c70d446450c/results/leaderboard.csv", "metric": "character_error_rate", "setup": "Riverborn Bangla STT Benchmark v1.0, tested September 2026 from Bangladesh: 1,001 BanSpeech utterances (77 from each of 13 domains, 50.0 min, 43,411 reference characters), batch whole-file requests with default settings plus each provider's Bangla language option, no custom vocabulary. Identical normalization (NFC, zero-width removal, danda/punctuation removal, Bangla digits to ASCII, lowercase Latin, whitespace collapse). Corpus CER (primary metric) = total character edits / total reference characters; errored clips excluded (coverage >= 99.4%); 95% bootstrap CI over clips (10,000).", "sample_count": 1001, "caveats": [ "Short utterances (median 2.1 s); read/broadcast-dominated, no dialects.", "Point-in-time hosted APIs; single run.", "CER primary because Bangla word segmentation is inconsistent; WER retained as companion metadata only." ] }, { "id": "perle-cs4-mean-wer", "name": "Perle code-switching benchmark · 4-pair mean WER", "category": "stt", "kind": "err", "url": "https://arxiv.org/pdf/2605.19069v3", "metric": "word_error_rate", "setup": "arXiv:2605.19069v3 Table 4: production APIs called in May 2026 on 300 selected code-switched utterances per pair (Egyptian Arabic-English, Saudi Arabic-English, Persian-English, German-English; 1,200 total; voice-actor recordings). No language hint for Scribe v2 (model_id=scribe_v2), gpt-4o-transcribe (temperature=0, response_format=text) or Chirp 3 (language_codes=['auto'], us region). Normalization: lowercase, punctuation removed, whitespace collapsed; Arabic/Persian script preserved (no hamza/ta-marbuta/digit folding). jiwer WER; 'Mean WER' as printed in Table 4.", "sample_count": 1200, "caveats": [ "Aggregation of 'Mean WER' (per-utterance vs per-pair mean) not stated precisely.", "Script/transliteration choices penalised by WER (paper reports BERTScore too; not imported).", "Azure (CLID) not a catalogue identity; Deepgram Nova-3 measured on German-English only and excluded." ] }, { "id": "lmarena-image-edit", "name": "Arena image edit", "kind": "elo", "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset", "category": "image", "task": "image-editing", "setup": { "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset", "version": "image_edit config, latest split, category overall, revision 6931a5ef0714c8d4edf09d79998fef6674ed4e09", "metric": "Arena Score (Bradley-Terry rating from pairwise human votes); 95% CI retained in note", "comparability": "Separate from text_to_image and from the multi_image_edit category. Rows keep Arena's exact entry label; resolution/quality/web-search variants are not merged unless an existing text_to_image mapping already uses the same label." }, "reported_scale": "Elo", "benchmark_family": "lmarena-image-edit" }, { "id": "lmarena-video-edit", "name": "Arena video edit", "kind": "elo", "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset", "category": "video", "task": "video-editing", "setup": { "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset", "version": "video_edit config, latest split (published 2026-08-27), category overall, revision 6931a5ef0714c8d4edf09d79998fef6674ed4e09", "metric": "Arena Score (Bradley-Terry rating from pairwise human votes); 95% CI retained in note", "comparability": "Video-to-video editing; not T2V or I2V. Small vote counts (463-13,724) give wide intervals." }, "reported_scale": "Elo", "benchmark_family": "lmarena-video-edit" }, { "id": "genexam-relaxed", "name": "GenExam (relaxed overall, GPT-5 judge)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2509.14232v5#S3.T2", "task": "text-to-image", "setup": { "url": "https://github.com/OpenGVLab/GenExam", "version": "GenExam report v5 (14 May 2026), Table 2; 1,000 exam-style prompts across 10 subjects", "metric": "Published Overall relaxed score: weighted average of semantic correctness (0.7), spelling, logical consistency and readability (0.1 each); 0-100 divided by 100.", "protocol": "Benchmark authors' runs with each model's default T2I configuration; GPT-5 MLLM judge (reasoning effort low) with per-sample scoring points and ground-truth reference image.", "comparability": "Relaxed score only. The strict score (all-or-nothing, 0 for 12 of 19 rows) is the same run and is not recorded separately; do not mix with Gemini-3-Flash evaluator ablations." }, "reported_scale": "0-100", "benchmark_family": "genexam" }, { "id": "t2i-reasonbench-overall", "name": "T2I-ReasonBench (overall)", "category": "image", "kind": "acc", "chance": 0, "url": "https://aclanthology.org/2026.findings-acl.433/", "task": "text-to-image", "setup": { "url": "https://github.com/KaiyueSun98/T2I-ReasonBench", "version": "Findings of ACL 2026 version (Table 2), 800 prompts; not the arXiv v1 table", "metric": "Published Overall T2I-ReasonScore (mean of Idiom Interpretation, Textual Image Design, Entity-Reasoning and Scientific-Reasoning), 0-100 divided by 100.", "protocol": "Original benchmark prompts sent directly to each model with default generation settings; question-criterion pairs from GPT-5.1 (Idiom, Textual, Scientific) and Gemini-2.5-pro (Entity); Gemini-2.5-pro MLLM judge.", "comparability": "Do not mix with Table 3 LLM-rewritten-prompt scores or with arXiv v1 values, which used an earlier evaluator configuration." }, "reported_scale": "0-100", "benchmark_family": "t2i-reasonbench" }, { "id": "tiif-bench-v3-short", "name": "TIIF-Bench v3 (short prompts, TIIF-Evaluator)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2506.02161v3#S4.T2", "task": "text-to-image", "setup": { "url": "https://github.com/A113N-W3I/TIIF-Bench", "version": "TIIF-Bench report v3 (12 Jul 2026) Table 2; prompt split (5,000-prompt full set vs 554-prompt testmini) not stated for this table", "metric": "Published Overall (short prompts): average over all dimensions and difficulty levels, text GNED transformed so higher is better; 0-100 divided by 100.", "protocol": "Benchmark authors' runs; attribute-level yes/no checklist verified by the trained TIIF-Evaluator; latest available model versions at evaluation time unless stated.", "comparability": "Different evaluator and possibly different split from the Mage-Flow-reported tiif-testmini-short/long tracks. Do not pool. Long-prompt Overall is a correlated track of the same run and is not recorded as a separate benchmark." }, "reported_scale": "0-100", "benchmark_family": "tiif" }, { "id": "genai-bench-vqascore", "name": "GenAI-Bench (VQAScore overall)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2506.02161v3#S4.T3", "task": "text-to-image", "setup": { "url": "https://github.com/linzhiqiu/t2v_metrics", "version": "GenAI-Bench complete prompt set (basic + advanced)", "metric": "Published Overall VQAScore average over basic and advanced prompts; already 0-1.", "protocol": "Run by the TIIF-Bench authors with GenAI-Bench's official evaluation protocol (Sec. 4.2).", "comparability": "Automatic VQAScore, not GenAI-Bench human ratings." }, "reported_scale": "0-1", "benchmark_family": "genai-bench" }, { "id": "wise-wiscore-overall", "name": "WISE (WiScore overall, GPT-4o judge)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2503.07265v4#S4.T1", "task": "text-to-image", "setup": { "url": "https://github.com/PKU-YuanGroup/WISE", "version": "WISE original 1,000 prompts (25 sub-domains), arXiv v4 Table 1", "metric": "Published Overall WiScore (weighted average across Cultural, Time, Space, Biology, Physics, Chemistry); WiScore = (0.7 consistency + 0.2 realism + 0.1 aesthetics)/2, already on 0-1.", "protocol": "Original (not rewritten) prompts; official default configuration per model with fixed seed; GPT-4o-2024-05-13 judge.", "comparability": "Not the rewritten-prompt Table 2, not the WISE-Verified/Qwen-judge leaderboard in the repo, not the gemini-3.1-flash-lite sensitivity runs." }, "reported_scale": "0-1", "benchmark_family": "wise" }, { "id": "sci-vbench-testmini-human", "name": "Sci-VBench testmini (expert human rating)", "category": "video", "kind": "acc", "chance": 0.2, "url": "https://arxiv.org/html/2608.09873v1#S5.T4", "task": "text-to-video", "setup": { "url": "https://arxiv.org/html/2608.09873v1#S5.T4", "version": "Sci-VBench v1 (arXiv 2608.09873v1), testmini split, 150 examples", "metric": "Human Eval Avg.: unweighted mean of four expert 1–5 integer ratings (Low-level Perceptual Fidelity, Spatiotemporal Consistency, Prompt Grounding, Scientific and Causal Correctness) against per-example rubrics. Stored as value/5; chance 0.2 maps the rubric floor (1) to zero.", "protocol": "Verbatim benchmark prompts, each model's default configuration; duration closest to 15 s (Veo-3.1 8 s, Gemini-Omni-Flash 10 s, CogVideoX1.5-5B 5 s); every testmini video rated by a domain expert (Section 4.3). Exact API ids / HF repos in Table 6.", "limitations": "Provider filters rejected 6 Gemini-Omni-Flash and 2 Seedance-2.0 prompts; their averages use successful generations. Automatic judge columns come from the same videos and are not a separate study." }, "reported_scale": "1–5", "benchmark_family": "sci-vbench" }, { "id": "fepbench-if", "name": "FEPBench instruction faithfulness", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2606.05949v2#S4.T2", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2606.05949v2#S4.T2", "version": "FEPBench v2 (arXiv 2606.05949v2, 2026-06-05); 1,300 natural-science illustrations; free-form and structured prompts", "metric": "Table 2 IF (instruction faithfulness) mean over visual, text, relation and layout instruction atoms, both prompt regimes; 0–1 as published.", "protocol": "Identical prompts for all models, no model-specific prompt engineering or manual selection; API models at listed sizes; open models 50 steps, guidance 5.0 (Table 6); tested April 2026; GPT-5.4 and Qwen3.5-397B-A17B evaluators; human-alignment study on 1,080 figures.", "comparability": "The benchmark has no single overall column. IF is used because the RE and SP means are further dimensions of the same experiment; importing all three would count one study three times." }, "reported_scale": "0–1", "benchmark_family": "fepbench" }, { "id": "edit-compass-en-overall", "name": "Edit-Compass (EN, overall)", "category": "image", "kind": "acc", "chance": 0.2, "url": "https://arxiv.org/html/2605.13062v1#S4.T3", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2605.13062v1#A4", "version": "Edit-Compass v1 (arXiv 2605.13062v1), English instructions, Table 3", "metric": "Published Overall (AVG) column. Per sample: s_min if min(IA,VC,VQ) <= 1, else weighted geometric mean of Instruction Awareness, Visual Consistency and Visual Quality (weights 0.4/0.4/0.2 base tasks, 0.5/0.3/0.2 world-knowledge, 0.6/0.2/0.2 algorithmic reasoning). Judge Gemini-3.1-Pro, 1–5 scale. Stored as reported/5 (chance 0.2 = floor of 1/5).", "protocol": "Six task categories (General, Dynamic Manipulation, World Knowledge, Visual Reasoning, Multi-Image, Complex); models without multi-image support have no Multi-Image cells ('-'); closed models via official APIs; open models at default settings (not further specified).", "limitations": "Single MLLM judge; per-model sampler/resolution not tabulated; Multi-Image category absent for single-image models, so Overall covers different task subsets for those models. Chinese-instruction track (Table 4) is a separate, correlated track and is not imported." }, "reported_scale": "1–5", "benchmark_family": "edit-compass" }, { "id": "geditbench-v2-overall-elo", "name": "GEditBench v2 (overall Elo)", "category": "image", "kind": "elo", "url": "https://arxiv.org/html/2603.28547v1#S5.T3", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2603.28547v1#S5.SS2", "version": "GEditBench v2 (arXiv 2603.28547v1), Table 3 OpenEdit leaderboard", "metric": "Overall Elo from one Bradley–Terry model fitted on aggregated pairwise comparisons across Instruction Following, Visual Quality (GPT-4o 26-03-24 judge) and Visual Consistency (PVC-Judge); 95% CI from 1,000 bootstraps retained in notes.", "protocol": "1,200 real-world user queries over 23 tasks incl. open-set; default hyperparameters; closed models accessed on the dated snapshot shown in the label.", "limitations": "Automatic pairwise judges, not human votes. Closed models returned fewer valid samples (Nano Banana Pro 1,156, Seedream 4.5 1,190, GPT Image 1.5 1,081 of 1,200). Elo scale is relative to this 16-model pool; do not pool with LMArena or Artificial Analysis ratings." }, "reported_scale": "Elo", "benchmark_family": "geditbench-v2" }, { "id": "unicbench-en-overall", "name": "UnicBench (EN, overall)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2512.02790v1#S6.T4", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2512.02790v1#S6.SS1", "version": "UnicBench from UnicEdit-10M report v1 (arXiv 2512.02790v1), Table 4", "metric": "Published Overall-EN: per-sample geometric mean of the applicable 0–10 metrics (Instruction Following, Non-edit Consistency, Visual Quality, plus Reasoning Accuracy for reasoning tasks), judged by gpt-4.1-2025-04-14; a zero in any metric marks a failed edit. Stored as reported/10.", "protocol": "English prompts; models supporting Chinese were also run on CN prompts (separate, not imported). Closed models via their services; generation settings not tabulated.", "limitations": "Single GPT-4.1 judge. 'FLUX.1-Kontext' is listed under open-source models with no tier label; mapped to [dev], the only open-weight Kontext release." }, "reported_scale": "0–10", "benchmark_family": "unicbench" }, { "id": "tecci-human-overall-success", "name": "TECCI human overall success rate (IRCS+GGIS)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2606.01213v1#S4.T2", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2606.01213v1#S4.SS3", "version": "TECCI v1 (arXiv 2606.01213v1), Table 2, IRCS + GGIS columns", "metric": "Overall success rate: share of edits whose five-rater mean is >= 4.5 on all of Instruction Following, Image Consistency and Visual Quality (1–5 Likert). Reported %, stored /100.", "protocol": "1,315 image/instruction pairs (1,050 GGIS + 265 IRCS), five trained raters per output, low-correlation raters filtered; proprietary models via official APIs with default hyper-parameters (Nano Banana 2 = gemini-3.1-flash-image-preview, Nano Banana Pro = gemini-3-pro-image-preview).", "limitations": "Human subset of the benchmark, five models only; strict all-criteria threshold compresses scores. Evaluator (Google) made two of the five models." }, "reported_scale": "0–100", "benchmark_family": "tecci" }, { "id": "skyreels-bench-t2v-human", "name": "SkyReels-Bench T2V (human rating)", "category": "video", "kind": "acc", "chance": 0.2, "url": "https://arxiv.org/html/2504.13074v3#S5.T5", "task": "text-to-video", "setup": { "url": "https://arxiv.org/html/2504.13074v3#S5.T5", "version": "SkyReels-Bench as reported in SkyReels-V2 v3", "metric": "Published Average of four human-rated dimensions (Instruction Adherence, Consistency, Visual Quality, Motion Quality), 1–5 scale; stored as value/5 with chance 0.2 mapping the scale floor to zero.", "protocol": "Text-to-video; 1,020 prompts; panel of 20 professional evaluators scoring each dimension 1–5 (Table 4 / Appendix A rubric); all models evaluated under default settings with consistent resolutions, no post-generation filtering.", "limitations": "In-house benchmark by a competing model developer; per-model API snapshots, resolutions and durations not reported." }, "reported_scale": "1–5", "benchmark_family": "skyreels-bench" }, { "id": "skyreels-bench-i2v-human", "name": "SkyReels-Bench I2V (human rating)", "category": "video", "kind": "acc", "chance": 0.2, "url": "https://arxiv.org/html/2504.13074v3#S6.T7", "task": "image-to-video", "setup": { "url": "https://arxiv.org/html/2504.13074v3#S6.T7", "version": "SkyReels-Bench as reported in SkyReels-V2 v3", "metric": "Published Average of four human-rated dimensions (Instruction Adherence, Consistency, Visual Quality, Motion Quality), 1–5 scale; stored as value/5 with chance 0.2 mapping the scale floor to zero.", "protocol": "Image-to-video (adds first-frame fidelity to Consistency); 1,020 prompts; panel of 20 professional evaluators scoring each dimension 1–5 (Table 4 / Appendix A rubric); all models evaluated under default settings with consistent resolutions, no post-generation filtering.", "limitations": "In-house benchmark by a competing model developer; per-model API snapshots, resolutions and durations not reported." }, "reported_scale": "1–5", "benchmark_family": "skyreels-bench" }, { "id": "vbench-t2v-total-long-prompts", "name": "VBench T2V total (long-prompt version, SkyReels-V2 runs)", "category": "video", "kind": "acc", "chance": 0, "url": "https://github.com/Vchitect/VBench", "task": "text-to-video", "benchmark_family": "vbench", "reported_scale": "0–100", "setup": { "metric": "Published VBench 1.0 Total Score, divided by 100", "version": "VBench 1.0 T2V, benchmark's longer (extended) prompt version", "evaluation_scope": "Skywork's own inference of each baseline following the baseline's recommended inference setting (SkyReels-V2 v3 Section 5.2).", "comparability": "Long-prompt version; not the standard-prompt total (vbench-t2v-total), VBench-Long, VBench-I2V or VBench-2.0. Quality/Semantic sub-scores not imported.", "source_protocol": "https://arxiv.org/html/2504.13074v3#S5.SS2.p2.1" } }, { "id": "videophy2-human-joint", "name": "VideoPhy-2 (human, joint SA & PC)", "category": "video", "kind": "acc", "chance": 0, "url": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md", "task": "text-to-video", "reported_scale": "0–100", "benchmark_family": "videophy2", "setup": { "url": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/VIDEOPHY2/README.md", "version": "VideoPhy-2 human leaderboard, 'All' column, repository commit 01d366c", "metric": "Joint performance: percentage of test prompts whose video has semantic adherence SA>=4 and physical commonsense PC>=4 under human annotation; divided by 100.", "protocol": "Action-centric VideoPhy-2 test prompts; human annotation by the benchmark authors.", "limitations": "Hard subset (CogVideoX-5B reference) not imported. Sora row was evaluated manually on a 60-prompt subset through its UI and is not imported. Generation settings per model are not stated in the README." } }, { "id": "videophy-autoeval-avg", "name": "VideoPhy (auto-evaluator average)", "category": "video", "kind": "acc", "chance": 0, "url": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md", "task": "text-to-video", "reported_scale": "0–100", "benchmark_family": "videophy", "setup": { "url": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md", "version": "VideoPhy-1 automatic leaderboard, repository commit 01d366c", "metric": "Avg. of the percentage of prompts with physical commonsense PC=1 and semantic adherence SA=1, judged by the authors' VideoCon-Physics auto-evaluator; divided by 100.", "protocol": "VideoPhy-1 test prompts; authors' own auto-evaluator run on generated videos.", "limitations": "Integer percentages; authors note the evaluator under-rates some closed models (Pika). Not VideoPhy-2 and not the VideoPhy-1 human leaderboard." } }, { "id": "v-reasonbench-pass5-avg", "name": "V-ReasonBench (average pass@5)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2511.16668v1#S3.T2", "task": "image-to-video", "reported_scale": "0–100", "benchmark_family": "v-reasonbench", "setup": { "url": "https://arxiv.org/html/2511.16668v1#S3.T2", "version": "V-ReasonBench v1 (arXiv 2511.16668v1)", "metric": "Published Average of four reasoning-dimension pass@5 scores (structured problem-solving, spatial cognition, pattern-based inference, physical dynamics); divided by 100.", "protocol": "Initial image + identical prompt per task; five ~5 s videos per prompt, default parameters, 720p/768p when supported, 16:9; last-frame answer verification (human-alignment check in Section 4.3).", "limitations": "Chain-of-frame reasoning, not general visual quality. Per-model API snapshot not stated." } }, { "id": "vwg-bench-overall", "name": "VWG-Bench think-with-video reasoning (overall)", "category": "video", "kind": "acc", "chance": 0.2, "url": "https://arxiv.org/html/2609.11242v1#S4.T2", "task": "image-to-video", "reported_scale": "1–5", "benchmark_family": "vwg-bench", "setup": { "url": "https://arxiv.org/html/2609.11242v1#S4.T2", "version": "VWG-Bench, arXiv 2609.11242v1 Table 2 baseline diagnostic", "metric": "Published Overall (mean of video quality, progress consistency, rule following, progress goal, last-frame goal) on a 1–5 scale from a Gemini-3.0-Pro judge; stored as value/5 with chance 0.2.", "protocol": "Image + user prompt (I2V); three generations per pair, scores averaged; no prompt extension for any model; human-agreement check in Table 4 (Spearman 0.69–0.80).", "limitations": "Several authors are from the Kling Team (Kuaishou); Kling rows are marked lab. Per-model resolution/duration not stated." } }, { "id": "worldjen-human-bt", "name": "WorldJen human preference (Bradley-Terry)", "category": "video", "kind": "elo", "url": "https://arxiv.org/html/2605.03475v2#S5.T1", "task": "text-to-video", "reported_scale": "Elo", "benchmark_family": "worldjen", "setup": { "url": "https://arxiv.org/html/2605.03475v2#S5.T1", "version": "WorldJen v2 (arXiv 2605.03475v2), 50 enhanced prompts × 6 models = 300 videos", "metric": "Human Bradley-Terry MLE rating on the Elo scale (400·log10; 1.49 BT log-odds = 259 points in Table 1); 95% bootstrap CI retained in note.", "protocol": "One video per (model, prompt); default settings, no prompt-specific tuning; commercial models via fal.ai (generated March 2026), Wan 2.1 1.3B locally; Veo 3.1 Fast ≈8 s, others ≈5 s; 2,696 pairwise votes from 7 annotators, 100% pair coverage.", "limitations": "Six-model pool; within-tier differences not significant. Ratings are relative to this pool only and are not on Arena's scale." } }, { "id": "t2vphysbench-human-avg", "name": "T2VPhysBench (human physical consistency)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2505.00337v1#S4.T2", "task": "text-to-video", "reported_scale": "0–1", "benchmark_family": "t2vphysbench", "setup": { "url": "https://arxiv.org/html/2505.00337v1#S4.T2", "version": "T2VPhysBench v1 (arXiv 2505.00337v1), original (non-counterfactual) prompts", "metric": "Published Avg. Score over Newton, conservation and phenomenon principles; each video graded by three annotators into four levels mapped to [0,1].", "protocol": "Fully manual evaluation following VideoPhy; scores averaged over prompts and annotators.", "limitations": "Small prompt set per principle; generation settings and dates per model not reported. Counterfactual/hint experiments not imported." } }, { "id": "qc-t2i-bench-en-hcq", "name": "QC-T2I-Bench (English, HCQ total)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2608.24112v1#Sx3.T2", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2608.24112v1#Sx3.T2", "version": "QC-T2I-Bench v1 (arXiv 2608.24112v1, 2026-08-25), English prompt set", "metric": "Hierarchy-Constrained Question aggregation (HCQ) Total: atomic yes/no questions with Davidsonian-scene-graph dependency validity, equally averaged over five capability views (entities, text, attributes, relations, high-level semantics); percent divided by 100.", "protocol": "Official defaults with configurable prompt enhancement disabled; every model generates the English and Chinese sets; text rendering scored with Qwen3-VL-30B OCR matching.", "comparability": "English and Chinese leaderboards are computed independently; only English is imported to count the experiment once." }, "reported_scale": "0–100", "benchmark_family": "qc-t2i-bench" }, { "id": "imagetime-prompt-only", "name": "ImageTime (prompt-only, overall)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2606.10620v1#S5.T5", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2606.10620v1#S5.T5", "version": "ImageTime v1 (arXiv 2606.10620v1, 2026-06-09); 750 prompt-only motion-sheet cases (375 action concepts x 2 variants)", "metric": "Overall mean: balanced average of capability-score (C) and diagnostic-score (D) means, integer 0–10 scales judged by GPT-5.5 with conservative score capping; divided by 10.", "protocol": "Each model generates an ordered 2x2 motion sheet from the text prompt only (no reference image); same 750 prompts for every model; single GPT-5.5 judging pipeline.", "comparability": "Reference-guided (scaffold) settings and category breakdowns are not imported. Spatiotemporal consistency within a single generated image, not video." }, "reported_scale": "0–10", "benchmark_family": "imagetime" }, { "id": "gebench-en-ge-score", "name": "GEBench GUI state generation (English GE-Score)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2602.09007v2#S3.T1", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2602.09007v2#S3.T1", "version": "GEBench v2 (arXiv 2602.09007v2, 2026-02-10); 700 samples, five task categories (single-step, multi-step, fiction-app, real-app, grounding)", "metric": "GE-Score: rubric scores 0–5 on goal achievement, interaction logic, content consistency, UI plausibility and visual quality, normalised to 0–100 and averaged; English subset; divided by 100.", "protocol": "Models generate the next GUI screen(s) from a screenshot and instruction; official default configurations; evaluation run three times; cross-evaluated by Gemini-3-Flash, GPT-4o and Qwen3-VL-235B (Table 1 aggregation across judges not stated).", "comparability": "Chinese subset is a related track and is not imported. Image-conditioned GUI transition, not free text-to-image." }, "reported_scale": "0–100", "benchmark_family": "gebench" }, { "id": "imagenworld-tig-human", "name": "ImagenWorld text-guided generation (human overall)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2603.27862v1#S5.T3", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2603.27862v1#S5.T3", "version": "ImagenWorld v1 (arXiv 2603.27862v1, 2026-03-29), Text-guided Image Generation (TIG) task", "metric": "Human-annotated overall (O_Human): mean of Prompt Relevance, Aesthetic Quality, Content Coherence and Artifact ratings, each a 5-point Likert score rescaled to [0,1], three annotators per image.", "protocol": "Default configurations from papers/official releases; open models seed 42; closed models via API; six real-world topical domains.", "comparability": "TIG and TIE use different inputs and prompt sets and are kept as separate benchmarks; the four reference-image tasks and VLM-judge columns are not imported." }, "reported_scale": "0–1", "benchmark_family": "imagenworld" }, { "id": "imagenworld-tie-human", "name": "ImagenWorld text-guided editing (human overall)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2603.27862v1#S5.T3", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2603.27862v1#S5.T3", "version": "ImagenWorld v1 (arXiv 2603.27862v1, 2026-03-29), Text-guided Image Editing (TIE) task", "metric": "Human-annotated overall (O_Human): mean of Prompt Relevance, Aesthetic Quality, Content Coherence and Artifact ratings, each a 5-point Likert score rescaled to [0,1], three annotators per image.", "protocol": "Default configurations from papers/official releases; open models seed 42; closed models via API; six real-world topical domains.", "comparability": "TIG and TIE use different inputs and prompt sets and are kept as separate benchmarks; the four reference-image tasks and VLM-judge columns are not imported." }, "reported_scale": "0–1", "benchmark_family": "imagenworld" }, { "id": "matharena-arxivmath-2025-12", "name": "MathArena ArXivMath December 2025 (17 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/arxivmath-1225", "family": "matharena-arxivmath", "description": "Research-level final-answer questions extracted by MathArena from arXiv papers of that month (fresh set each month, like AIME years). Plain prompt, no tools; answer in \\boxed{} parsed and compared to the gold answer. Score = mean accuracy over attempts. Separate monthly question sets are separate benchmarks." }, { "id": "matharena-arxivmath-2026-01", "name": "MathArena ArXivMath January 2026 (23 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/arxivmath-0126", "family": "matharena-arxivmath", "description": "Research-level final-answer questions extracted by MathArena from arXiv papers of that month (fresh set each month, like AIME years). Plain prompt, no tools; answer in \\boxed{} parsed and compared to the gold answer. Score = mean accuracy over attempts. Separate monthly question sets are separate benchmarks." }, { "id": "matharena-arxivmath-2026-02", "name": "MathArena ArXivMath February 2026 (32 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/arxivmath-0226", "family": "matharena-arxivmath", "description": "Research-level final-answer questions extracted by MathArena from arXiv papers of that month (fresh set each month, like AIME years). Plain prompt, no tools; answer in \\boxed{} parsed and compared to the gold answer. Score = mean accuracy over attempts. Separate monthly question sets are separate benchmarks." }, { "id": "matharena-arxivmath-2026-03", "name": "MathArena ArXivMath March 2026 (30 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/arxivmath-0326", "family": "matharena-arxivmath", "description": "Research-level final-answer questions extracted by MathArena from arXiv papers of that month (fresh set each month, like AIME years). Plain prompt, no tools; answer in \\boxed{} parsed and compared to the gold answer. Score = mean accuracy over attempts. Separate monthly question sets are separate benchmarks." }, { "id": "matharena-arxivmath-2026-04", "name": "MathArena ArXivMath April 2026 (40 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/arxivmath-0426", "family": "matharena-arxivmath", "description": "Research-level final-answer questions extracted by MathArena from arXiv papers of that month (fresh set each month, like AIME years). Plain prompt, no tools; answer in \\boxed{} parsed and compared to the gold answer. Score = mean accuracy over attempts. Separate monthly question sets are separate benchmarks." }, { "id": "matharena-arxivmath-2026-05", "name": "MathArena ArXivMath May 2026 (40 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/arxivmath-0526", "family": "matharena-arxivmath", "description": "Research-level final-answer questions extracted by MathArena from arXiv papers of that month (fresh set each month, like AIME years). Plain prompt, no tools; answer in \\boxed{} parsed and compared to the gold answer. Score = mean accuracy over attempts. Separate monthly question sets are separate benchmarks." }, { "id": "matharena-arxivmath-2026-06", "name": "MathArena ArXivMath June 2026 (48 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/arxivmath-0626", "family": "matharena-arxivmath", "description": "Research-level final-answer questions extracted by MathArena from arXiv papers of that month (fresh set each month, like AIME years). Plain prompt, no tools; answer in \\boxed{} parsed and compared to the gold answer. Score = mean accuracy over attempts. Separate monthly question sets are separate benchmarks." }, { "id": "matharena-brokenarxiv-2026-02", "name": "MathArena BrokenArXiv February 2026 (31 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/brokenarxiv-0226", "family": "matharena-brokenarxiv", "description": "Model is asked to prove a subtly false statement derived from an arXiv theorem; an LLM judge (Gemini 3.1 Pro, medium; MathArena arxiv_judge prompt, revised after March 2026) awards 0-2 points, 2 = explicitly recognises the statement is false. Score = mean points/2. Measures non-sycophantic mathematical verification. Separate monthly sets are separate benchmarks." }, { "id": "matharena-brokenarxiv-2026-03", "name": "MathArena BrokenArXiv March 2026 (56 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/brokenarxiv-0326", "family": "matharena-brokenarxiv", "description": "Model is asked to prove a subtly false statement derived from an arXiv theorem; an LLM judge (Gemini 3.1 Pro, medium; MathArena arxiv_judge prompt, revised after March 2026) awards 0-2 points, 2 = explicitly recognises the statement is false. Score = mean points/2. Measures non-sycophantic mathematical verification. Separate monthly sets are separate benchmarks." }, { "id": "matharena-brokenarxiv-2026-04", "name": "MathArena BrokenArXiv April 2026 (61 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/brokenarxiv-0426", "family": "matharena-brokenarxiv", "description": "Model is asked to prove a subtly false statement derived from an arXiv theorem; an LLM judge (Gemini 3.1 Pro, medium; MathArena arxiv_judge prompt, revised after March 2026) awards 0-2 points, 2 = explicitly recognises the statement is false. Score = mean points/2. Measures non-sycophantic mathematical verification. Separate monthly sets are separate benchmarks." }, { "id": "matharena-brokenarxiv-2026-05", "name": "MathArena BrokenArXiv May 2026 (50 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/brokenarxiv-0526", "family": "matharena-brokenarxiv", "description": "Model is asked to prove a subtly false statement derived from an arXiv theorem; an LLM judge (Gemini 3.1 Pro, medium; MathArena arxiv_judge prompt, revised after March 2026) awards 0-2 points, 2 = explicitly recognises the statement is false. Score = mean points/2. Measures non-sycophantic mathematical verification. Separate monthly sets are separate benchmarks." }, { "id": "matharena-brokenarxiv-2026-06", "name": "MathArena BrokenArXiv June 2026 (54 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/brokenarxiv-0626", "family": "matharena-brokenarxiv", "description": "Model is asked to prove a subtly false statement derived from an arXiv theorem; an LLM judge (Gemini 3.1 Pro, medium; MathArena arxiv_judge prompt, revised after March 2026) awards 0-2 points, 2 = explicitly recognises the statement is false. Score = mean points/2. Measures non-sycophantic mathematical verification. Separate monthly sets are separate benchmarks." }, { "id": "matharena-arxivlean-2026-03", "name": "MathArena ArXivLean March 2026 (41 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/arxivlean-0326", "family": "matharena-arxivlean", "description": "Prove formal Lean 4 (v4.29.0, mathlib) statements from March 2026 arXiv papers with MathArena's Lean tool loop (verify_lean, verify_submission, add_to_file, loogle, lean_explore_search; max 200 tool calls); success = Lean-verified proof of the unmodified statement. One attempt per problem." }, { "id": "eqbench3-elo", "name": "EQ-Bench 3 (emotional intelligence, judge Elo)", "kind": "elo", "category": "llm", "url": "https://github.com/EQ-bench/eqbench3/blob/affc5a000bbdb965b807e09ca7875f8530fa8f56/data/canonical_leaderboard_elo_results.json.gz", "description": "Multi-turn role-play and transcript-analysis scenarios testing empathy, social dexterity and insight; an LLM judge compares transcripts pairwise and a TrueSkill solver (mu start 1200) produces the rating. Raw 'elo' field of the canonical leaderboard file, not the anchored elo_norm display scale." }, { "id": "galileo-agent-v1-tsq", "name": "Galileo Agent Leaderboard v1 (tool selection quality)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v1.csv", "description": "Mean Tool Selection Quality over 14 subsets sampled from BFCL v3 (multi-turn, irrelevance), tau-bench long-context, xLAM and ToolACE; TSQ is Galileo's ChainPoll LLM-judge metric (GPT-4o). Equal weight per subset." }, { "id": "galileo-agent-v2-action-completion", "name": "Galileo Agent Leaderboard v2 (action completion)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/galileo-ai/agent-leaderboard/blob/8158b75ae542c2e847ff6d517a0b66caa704b19a/results_v2.csv", "description": "Average Action Completion over simulated multi-turn enterprise scenarios in banking, healthcare, insurance, investment and telecom (synthetic tools/personas, simulated user); AC judged by Galileo's LLM metric." }, { "id": "simple-evals-mmlu", "name": "MMLU (simple-evals, zero-shot CoT)", "kind": "acc", "category": "llm", "url": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md", "description": "Original 57-subject MMLU, OpenAI simple-evals zero-shot chain-of-thought prompt. OpenAI lab-reported (own models).", "chance": 0.25 }, { "id": "simple-evals-gpqa-diamond", "name": "GPQA Diamond (simple-evals)", "kind": "acc", "category": "llm", "url": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md", "description": "GPQA default 'diamond' variant in simple-evals gpqa_eval.py, zero-shot CoT, answer regex tweak. OpenAI lab-reported (own models).", "chance": 0.25 }, { "id": "simple-evals-math500", "name": "MATH-500 (simple-evals)", "kind": "acc", "category": "llm", "url": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md", "description": "MATH-500 split for models released on or after o1 (README footnote 6), zero-shot CoT. OpenAI lab-reported (own models)." }, { "id": "simple-evals-humaneval", "name": "HumanEval (simple-evals)", "kind": "acc", "category": "llm", "url": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md", "description": "Original HumanEval pass@1 with simple-evals sampler. OpenAI lab-reported (own models)." }, { "id": "simple-evals-mgsm", "name": "MGSM (simple-evals)", "kind": "acc", "category": "llm", "url": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md", "description": "Multilingual grade-school math, simple-evals zero-shot CoT. OpenAI lab-reported (own models)." }, { "id": "simple-evals-drop-f1", "name": "DROP F1, 3-shot (simple-evals)", "kind": "acc", "category": "llm", "url": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md", "description": "DROP reading comprehension F1 (0-1), 3-shot, simple-evals. OpenAI lab-reported (own models)." }, { "id": "simple-evals-simpleqa", "name": "SimpleQA (simple-evals)", "kind": "acc", "category": "llm", "url": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md", "description": "Fraction correct on SimpleQA, no browsing. OpenAI lab-reported (own models)." }, { "id": "arena-hard-v2-hard-prompt-sc-gemini25", "name": "Arena-Hard v2.0 hard prompts (style control, Gemini-2.5 judge)", "kind": "acc", "category": "llm", "url": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md", "chance": 0, "description": "500 hard real-world prompts; pairwise win rate against o3-mini-2025-01-31 judged by Gemini-2.5 with style control (official configuration). Distinct from the arena-hard-v2 aggregate reported in model cards." }, { "id": "arena-hard-v2-creative-writing", "name": "Arena-Hard v2.0 creative writing (GPT-4.1 + Gemini-2.5 judges)", "kind": "acc", "category": "llm", "url": "https://github.com/lmarena/arena-hard-auto/blob/196f6b826783b3da7310e361a805fa36f0be83f3/README.md", "chance": 0, "description": "250 creative-writing prompts; win rate against gemini-2.0-flash-001 judged by an ensemble of GPT-4.1 and Gemini-2.5." }, { "id": "scicode-main-official", "name": "SciCode main problems (official leaderboard)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/scicode-bench/SciCode/blob/e3158ea011d4235245a547460d3688d7ccbf9900/README.md", "description": "Fraction of SciCode main problems fully solved (all subproblems pass), maintainers' runs; subproblem rate not also added." }, { "id": "judgemark-v2", "name": "Judgemark v2 (LLM-as-judge for creative writing)", "kind": "acc", "category": "llm", "url": "https://github.com/EQ-bench/Judgemark-v2/blob/2388448a0401c4d6ed5502ff18953adba944a255/results/scores.csv", "chance": 0, "description": "Calibrated Judgemark score (0-100, divided by 100): separability, consistency and agreement of the model's numeric creative-writing judgements." }, { "id": "arcprize-arc-agi-2-public-eval", "name": "ARC-AGI-2 public evaluation set (ARC Prize model_baseline, pass@2)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval", "description": "ARC Prize Foundation runs of API models on the 120-task ARC-AGI-2 public evaluation set with the open model_baseline harness; 2 attempts per test output, a pair counts if either attempt is exactly correct, task score = fraction of its test pairs solved, overall = mean over all tasks. Public set, so contamination is possible." }, { "id": "arcprize-arc-agi-1-public-eval", "name": "ARC-AGI-1 public evaluation set (ARC Prize model_baseline, pass@2)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval", "description": "ARC Prize Foundation runs of API models on the 400-task ARC-AGI-1 public evaluation set with the open model_baseline harness; 2 attempts per test output, a pair counts if either attempt is exactly correct, task score = fraction of its test pairs solved, overall = mean over all tasks. Public set, so contamination is possible." }, { "id": "vector-inspect-gpqa-diamond", "name": "GPQA Diamond (Vector Institute, Inspect Evals)", "kind": "acc", "chance": 0.25, "category": "llm", "url": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json", "description": "Vector Institute leaderboard run of inspect_evals `gpqa_diamond` (198 questions), metric accuracy. Evaluator-specific protocol; not pooled with other GPQA Diamond series." }, { "id": "vector-inspect-mmlu-pro", "name": "MMLU-Pro (Vector Institute, Inspect Evals)", "kind": "acc", "chance": 0.1, "category": "llm", "url": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json", "description": "Vector Institute leaderboard run of inspect_evals `mmlu_pro`, metric accuracy." }, { "id": "vector-inspect-math", "name": "MATH (Vector Institute, Inspect Evals)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json", "description": "Vector Institute leaderboard run of inspect_evals `math` task (Hendrycks MATH test set as implemented by Inspect Evals), metric accuracy. Subset (full vs MATH-500) as implemented by inspect_evals at run time; kept separate from other MATH ids." }, { "id": "vector-inspect-humaneval", "name": "HumanEval (Vector Institute, Inspect Evals)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/vector-institute/eval-leaderboard/blob/f348507a79f56dae8624b525574cb326359a20c8/data/results.json", "description": "Vector Institute leaderboard run of inspect_evals `humaneval`, metric mean (pass@1)." }, { "id": "bridge-clinical-zeroshot-avg", "name": "BRIDGE clinical text, zero-shot average (87 tasks)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/YLab-Open/BRIDGE-Medical-Leaderboard/blob/579abc27d8659fce3ca357ee9108d4e279fbeb38/leaderboards/Zero-Shot_leaderboard.json", "description": "BRIDGE (Wu et al., Nature Biomedical Engineering 2026) leaderboard 'Average Performance' under the zero-shot strategy: unweighted mean of each of 87 real-world clinical text tasks' primary metric (accuracy/F1/ROUGE etc., 9 languages), divided by 100. Verified at the pinned revision that every model has all 87 task scores and the average equals their mean. Few-shot and CoT leaderboards deliberately not imported (same tasks)." }, { "id": "robloxqa-v2-mc", "name": "RobloxQA v2.0 multiple choice (Torpedo Software)", "kind": "acc", "chance": 0.25, "category": "llm", "url": "https://huggingface.co/datasets/TorpedoSoftware/RobloxQA-v2.0", "description": "3,000-question test split, four options; score = correct / 3000 from the leaderboard's raw counts. Questions were generated by Qwen3.6-35B-A3B and verified by Claude Sonnet 5 (possible family bias). Freeform split (grading method not documented) not imported." }, { "id": "luau-leetcode-full-pass", "name": "Luau Leetcode fully solved (Torpedo Software)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/TorpedoSoftware/LuauLeetcode", "description": "Single-shot Luau code generation on 208 Leetcode problems (47 easy/97 medium/64 hard), Jest-Lua unit tests; score = problems with all tests passing (fullPass summed over difficulties) / 208. The site's weighted 'coding subscore' is not used." }, { "id": "webapp1k-react-pass1", "name": "WebApp1K-React pass@1 (onekq-ai)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/onekq-ai/WebApp1K-React", "description": "1,000 React web-app problems (Jest tests; arXiv 2408.00019/2409.05177). Score = mean over problems of Successes/Runs (the Space's pass@1 estimator)." }, { "id": "webapp1k-duo-react-pass1", "name": "WebApp1K-Duo-React pass@1 (onekq-ai)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/onekq-ai/WebApp1K-Duo-React", "description": "1,000 harder two-feature React problems (arXiv 2409.13773). Score = mean over problems of Successes/Runs (the Space's pass@1 estimator)." }, { "id": "mideind-icelandic-average", "name": "Icelandic LLM Leaderboard average (Miðeind, Sept 2026 suite)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/spaces/mideind/icelandic-llm-leaderboard/blob/f8e42e15da1b985242bbb29eff139ec62405b53b/leaderboard.json", "description": "Flat mean of 7 columns over 10 Icelandic tests (WikiQA-IS; recall = song lyrics + famous phrases; terminology = textiles + flora/fauna; generative quality; GEC; case government; idioms = cloze + meanings). WikiQA, textiles and idiom meanings graded by an LLM judge (GPT-5.6 Luna); others exact match/overlap. Formula verified against every model at the pinned revision. Suite introduced 6 Sept 2026 (archived v1 board not imported)." }, { "id": "soohak-mini-avg3", "name": "SOOHAK-Mini (702 problems, Avg@3)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2605.09063v3", "description": "Research-level mathematics problems curated by mathematicians; Avg@3 = mean accuracy over 3 samples per problem. Soohak authors' own runs (Table 2). Distinct from the Challenge split.", "chance": 0 }, { "id": "soohak-challenge-avg3", "name": "SOOHAK Challenge (340 problems, Avg@3)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2605.09063v3", "description": "Harder Challenge split of SOOHAK; Avg@3 = mean accuracy over 3 samples. Soohak authors' own runs (Table 2). Refusal split not imported.", "chance": 0 }, { "id": "mathconstraint-329-notools", "name": "MathConstraint full release, no tools (329 instances)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2605.08498v1", "description": "Verifier-accepted accuracy on 329 generated combinatorial/constraint reasoning instances without tools (Table 2, Accuracy column). Georgia Tech authors' own runs.", "chance": 0 }, { "id": "dseffi-bench-pass", "name": "DSEffi-Bench Pass (1,000 data-science tasks, greedy)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2608.30248v1", "description": "Fraction of 1,000 StackOverflow-derived data-science coding tasks solved (tests pass), greedy decoding T=0.0 (Table 2 Overall Pass). Efficiency scores not imported.", "chance": 0 }, { "id": "finsheet-bench-acc", "name": "FinSheet-Bench accuracy (501 questions, 24 spreadsheets)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2603.07316v1", "description": "Zero-shot accuracy on 501 questions over 24 private-equity fund spreadsheets (Table 4, Acc. on processed questions; coverage 100% for imported rows).", "chance": 0 }, { "id": "terminal-bench-2-terminus2", "name": "Terminal-Bench 2.0 with Terminus 2 agent (benchmark authors' runs)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2601.11868v1", "description": "Task resolution rate on Terminal-Bench 2.0 (89 tasks) using the Terminus 2 scaffold, run by the Terminal-Bench authors with provider-default reasoning (medium for OpenAI/Anthropic), >=5 trials per task. Kept separate from terminal-bench-2 because that id mixes scaffolds.", "chance": 0 }, { "id": "ale-openclaw-overall-pass", "name": "Agents' Last Exam overall full-pass rate (fixed OpenClaw + GUI-as-Tool)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2606.05405v2", "description": "Full-pass rate over all distinct ALE tasks (Near-Term 67, Full-Spectrum 55, Last-Exam 38) with the harness fixed to OpenClaw plus GUI-as-Tool (Table 1 model-comparison panel, Overall column).", "chance": 0 }, { "id": "medmarks-v-winrate", "name": "Medmarks-V weighted mean win rate (61 models, 71 configurations)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2605.01417v1", "description": "Weighted mean pairwise win rate across the verifiable Medmarks-V medical task suite, relative to the 71 evaluated configurations in the Medmarks paper (Table 4). A pool-relative score: 0.5 is the pool average.", "chance": 0 }, { "id": "hle-verified-text-full", "name": "HLE-Verified full set, text-only (Qwen Team runs)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2602.13964v4", "description": "Accuracy on HLE-Verified, the systematically verified and revised release of Humanity's Last Exam, text-only full set (Table 2). Not the original HLE; distinct from hle.", "chance": 0 }, { "id": "token-games-elo", "name": "The Token Games Elo (10-model puzzle-duel tournament)", "kind": "elo", "category": "llm", "url": "https://arxiv.org/html/2602.17831v2", "description": "Elo from match-level outcomes of 90 pairwise duels in which models propose and solve programming puzzles (Table 1, TTG Elo). Pool of 10 frontier models; highest thinking/reasoning mode enabled." }, { "id": "sciclaimeval-t1-pair-dev", "name": "NTCIR SciClaimEval Subtask 1 pair-accuracy (dev, single model)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2609.00654v1", "description": "Pair-accuracy for validating Supported/Refuted scientific claims against figures/tables (multimodal), dev set, honest per-sample single-model protocol without the pair prior (SciTrue Table 3, T1 Pair).", "chance": 0 }, { "id": "terminalworld-verified-terminus2", "name": "TerminalWorld-Verified (200 tasks) with Terminus-2", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2605.22535v1", "description": "Pass rate on the 200 human-verified real-world terminal tasks of TerminalWorld, Terminus-2 scaffold, provider-default reasoning, temperature 0.7 (Table 1 headline Pass Rate; tasks with harness errors count as failures).", "chance": 0 }, { "id": "mathnet-solve-test-micro", "name": "MathNet-Solve-Test micro-average accuracy (6,400 olympiad problems)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2604.18584v2", "description": "Accuracy on MathNet-Solve-Test olympiad problems graded against expert reference solutions (Table 3, Micro Avg.). MIT authors' runs.", "chance": 0 }, { "id": "aragen-v3-3c3h", "name": "AraGen-v3 3C3H score (Inception, Claude Sonnet 4 judge)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/aragen_v3_results.json", "description": "Inception's held-out AraGen-v3 Arabic generative benchmark (324 prompts: QA, reasoning, grammar, safety). 3C3H score (correctness, completeness, conciseness, helpfulness, honesty, harmlessness) judged by claude-sonnet-4-20250514, bootstrap CIs. Chat/instruct models only. Distinct from AraGen-12-24 and AraGen-03-25.", "chance": 0 }, { "id": "hindigen-v1-3c3h", "name": "HindiGen-v1 3C3H score (Inception, Claude 3.5 Sonnet judge)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/hindigen_v1_results.json", "description": "Inception HindiGen-v1 Hindi generative benchmark (QA, grammar, safety; 279 prompts), 3C3H score judged by claude-3.5-sonnet. Only rows judged by claude-3.5-sonnet are used; a row judged by claude-3.7-sonnet is excluded.", "chance": 0 }, { "id": "arabic-ifeval-inception-ar-prompt", "name": "Arabic IFEval prompt-level (Inception)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/inception42/Leaderboards/blob/bef41ddf1f6c39bb3a20ad13e518e28e4f2654fc/assets/results/ifeval_results.jsonl", "description": "Inception Arabic IFEval leaderboard, Arabic prompt-level instruction-following accuracy ('Ar Prompt-lvl'). Strict/loose checking not stated. English half of the leaderboard not imported.", "chance": 0 }, { "id": "polish-eq-bench-v2", "name": "Polish EQ-Bench v2 (SpeakLeash)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/speakleash/polish_eq-bench/blob/ec5fcb7c7b9b9ee2fdc3e56651e59e2eb12c5765/benchmark_results.csv", "description": "EQ-Bench v2 emotional-intelligence questions translated and localised into Polish (171 questions, benchmark version eq-bench_v2_pl), run by SpeakLeash. EQ-Bench score 0-100 divided by 100; it is a scaled distance score, not accuracy.", "chance": 0, "metric": "eq_bench_v2_score_div_100" }, { "id": "cptub-speakleash-avg", "name": "CPTUB average (SpeakLeash, 0-5 judge scale)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/speakleash/cptu_bench/blob/7f981eb09374a686f238138b61e8ba8744251991/data.json", "description": "Complex Polish Text Understanding Benchmark: implicatures (sentiment, language understanding, phraseology) and tricky questions, each graded 0-5 by an LLM judge (tricky questions: GPT-4o). Score = the Space's unweighted mean of the four category scores, divided by 5. A judge rating, not accuracy.", "chance": 0, "metric": "mean_judge_rating_div_5" }, { "id": "browsergym-genericagent-miniwob", "name": "BrowserGym MiniWoB (AgentLab GenericAgent)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard", "description": "MiniWoB success rate for ServiceNow AgentLab's GenericAgent (TMLR configuration flags) with the named LLM as backend, as published on the BrowserGym leaderboard. Only rows marked Original and followed_evaluation_protocol=Yes.", "chance": 0 }, { "id": "browsergym-genericagent-workarena-l1", "name": "BrowserGym WorkArena-L1 (AgentLab GenericAgent)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard", "description": "WorkArena-L1 success rate for ServiceNow AgentLab's GenericAgent (TMLR configuration flags) with the named LLM as backend, as published on the BrowserGym leaderboard. Only rows marked Original and followed_evaluation_protocol=Yes.", "chance": 0 }, { "id": "browsergym-genericagent-workarena-l2", "name": "BrowserGym WorkArena-L2 (AgentLab GenericAgent)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard", "description": "WorkArena-L2 success rate for ServiceNow AgentLab's GenericAgent (TMLR configuration flags) with the named LLM as backend, as published on the BrowserGym leaderboard. Only rows marked Original and followed_evaluation_protocol=Yes.", "chance": 0 }, { "id": "browsergym-genericagent-webarena", "name": "BrowserGym WebArena (AgentLab GenericAgent)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard", "description": "WebArena success rate for ServiceNow AgentLab's GenericAgent (TMLR configuration flags) with the named LLM as backend, as published on the BrowserGym leaderboard. Only rows marked Original and followed_evaluation_protocol=Yes.", "chance": 0 }, { "id": "open-pt-llm-avg9", "name": "Open PT-LLM Leaderboard average (9 Portuguese tasks)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard/blob/788e0e7aa84b7dee80fc3aa3960beafb9e508ec9/data/leaderboard.csv.gz", "description": "Unweighted mean of ENEM (3-shot acc), BLUEX (3-shot acc), OAB Exams (3-shot acc), ASSIN2 RTE (15-shot macro-F1), ASSIN2 STS (15-shot Pearson), FaQuAD NLI (15-shot macro-F1), HateBR (25-shot macro-F1), PT Hate Speech (25-shot macro-F1) and tweetSentBR (25-shot macro-F1), as published in the leaderboard 'average' column. Mixed metrics; not an accuracy. Chance 0.3264 is the leaderboard’s published random baseline for the average (data/metadata.json at the pinned revision: \"baselines\": {\"average\": 32.64}; about page: \"The baseline is the random model mean result for the task\").", "chance": 0.3264 }, { "id": "open-persian-llm-v2-avg21", "name": "Open Persian LLM Leaderboard v2 average (21 tasks)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/opll-org/Open-Persian-LLM-Leaderboard/blob/0167a4aa7f90054c5d24c3ec8fd2ea89f0a60365/leaderboard_data.jsonl", "description": "Part DP AI / AUT NLP Lab Persian leaderboard v2: unweighted mean of 21 Persian task accuracies (general knowledge, GSM8K, homographs, PIQA, proverbs, verb eval, Winogrande, ARC, Feqh, hallucination, Hellaswag, law, AUT MCQ, literature, BoolQ, reading comprehension, PartExpert, MMLU-Pro, Iranian social norms), reproducing the Space's 'Average Accuracy'.", "chance": 0 }, { "id": "mimic-cdm-avg4", "name": "MIMIC-CDM clinical decision making (4 pathologies)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/MIMIC-CDM/leaderboard/blob/43e1ef313f61a726cfa30655284333ce7cfa641c/data/leaderboards.json", "description": "Autonomous clinical decision making on MIMIC-IV abdominal cases: the model requests information and diagnoses appendicitis, cholecystitis, diverticulitis or pancreatitis. Mean diagnostic accuracy over the four pathologies.", "chance": 0 }, { "id": "mimic-cdm-fi-avg4", "name": "MIMIC-CDM full information (4 pathologies)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/MIMIC-CDM/leaderboard/blob/43e1ef313f61a726cfa30655284333ce7cfa641c/data/leaderboards.json", "description": "MIMIC-CDM full-information variant: all case information is given up front; mean diagnostic accuracy over appendicitis, cholecystitis, diverticulitis and pancreatitis. Distinct from the autonomous CDM setting.", "chance": 0 }, { "id": "libra-mini-2605", "name": "LIBRA Mini (Russian long context, May 2026 version)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/ai-forever/LIBRA-Leaderboard", "description": "ai-forever LIBRA May 2026 version, LIBRA Mini subset (ruBABILongQA3, ruSciPassageCount, LibrusecMHQA, LongContextMultiQ, ru2WikiMultihopQA, MatreshkaNames), exact match, 4k-128k contexts. Score = unweighted mean of the six dataset_total_score values, the Space's own Mini total.", "chance": 0 }, { "id": "rpc-bench-text-overall", "name": "RPC-Bench overall, text input (Z.ai)", "kind": "acc", "category": "llm", "url": "https://huggingface.co/spaces/zai-org/RPC-Bench/blob/e9313f371c1646c5c711581490e01fcf165ebfab/data/leaderboard.json", "description": "RPC-Bench research-paper comprehension, text-input configuration, dataset rpc-bench-test-v1, scored by eval.py@rpc-leaderboard-v1 (open QA judged by gpt-5-2025-08-07 and gemini-2.5-pro; claim verification exact match). Published 'overall' (informativeness) score divided by 100.", "chance": 0 }, { "id": "multi-swe-bench-7lang-magentless", "name": "Multi-SWE-bench, 7 languages (MagentLess scaffold)", "kind": "acc", "category": "llm", "url": "https://github.com/multi-swe-bench/experiments/tree/6a7d5566f62fa76f4192302cf763051b98e4facc/evaluation", "description": "Multi-SWE-bench verified splits for Java, TypeScript, JavaScript, Go, Rust, C and C++ (1,632 instances), resolved fraction summed over the seven languages, for the ByteDance Seed team's MagentLess (Agentless-derived) baseline runs. Python excluded.", "chance": 0 }, { "id": "swe-bench-pro-qwen-refined", "name": "SWE-bench Pro public set, Qwen-refined", "kind": "acc", "category": "llm", "description": "SWE-bench Pro public set after Qwen corrected some problematic tasks and re-evaluated all baselines on the refined benchmark (Qwen3.6/3.8 cards). Not the official public or commercial split; harness per note.", "url": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md" }, { "id": "skillsbench-qwen-78", "name": "SkillsBench 78-task subset (OpenCode, avg of 5)", "kind": "acc", "category": "llm", "description": "SkillsBench evaluated via OpenCode on 78 self-contained tasks (API-dependent tasks excluded), average of 5 runs, as reported in the Qwen3.6 cards. Distinct from the 87-task skillsbench-87 subset.", "url": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md" }, { "id": "claw-eval-avg", "name": "Claw-Eval average score", "kind": "acc", "category": "llm", "description": "Claw-Eval agent benchmark, mean score across trials as reported by the source.", "url": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md" }, { "id": "claw-eval-pass3", "name": "Claw-Eval Pass^3", "kind": "acc", "category": "llm", "description": "Claw-Eval agent benchmark, Pass^3 (task passed in all three trials) as reported by the source.", "url": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md" }, { "id": "qwenclawbench", "name": "QwenClawBench (Qwen internal)", "kind": "acc", "category": "llm", "description": "Qwen's internal real-user-distribution Claw agent benchmark; temp=0.6, 256K context. Lab-internal; only comparable within a Qwen source.", "url": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md" }, { "id": "mcpmark", "name": "MCPMark", "kind": "acc", "category": "llm", "description": "MCPMark MCP tool-use benchmark pass rate; MCP server versions and truncation per note.", "url": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md" }, { "id": "seccodebench", "name": "SecCodeBench", "kind": "acc", "category": "llm", "description": "SecCodeBench secure code generation benchmark; score as reported by the source.", "url": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md" }, { "id": "webarena-verified", "name": "WebArena-Verified", "kind": "acc", "category": "llm", "description": "WebArena-Verified browser-agent success rate with the official WebArena-Verified grader; scaffold per note.", "url": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md" }, { "id": "matharena-apex", "name": "MathArena Apex (pass@1)", "kind": "acc", "category": "llm", "description": "MathArena Apex hardest final-answer competition problems, pass@1 as reported by the source (lab-run unless stated).", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" }, { "id": "terminal-bench-3", "name": "Terminal-Bench 3.0", "kind": "acc", "category": "llm", "description": "Terminal-Bench version 3.0 pass@1; harness per note. Distinct from 2.0/2.1/4.0.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" }, { "id": "terminal-bench-4", "name": "Terminal-Bench 4.0", "kind": "acc", "category": "llm", "description": "Terminal-Bench version 4.0 pass@1; harness per note. Distinct from 2.x/3.0.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" }, { "id": "deepswe-v1-1", "name": "DeepSWE v1.1 (resolved)", "kind": "acc", "category": "llm", "description": "DeepSWE v1.1 software-engineering agent benchmark, resolved rate; harness (mini-SWE, Claude Code, ...) per note.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" }, { "id": "programbench-almost1", "name": "ProgramBench (Almost@1)", "kind": "acc", "category": "llm", "description": "ProgramBench whole-program generation, Almost@1 metric as reported by the source.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" }, { "id": "sec-bench-pro", "name": "SEC-Bench Pro (pass@1)", "kind": "acc", "category": "llm", "description": "SEC-Bench Pro security-engineering agent benchmark, pass@1; harness per note.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" }, { "id": "exploitgym", "name": "ExploitGym (pass@1)", "kind": "acc", "category": "llm", "description": "ExploitGym exploit-development agent benchmark, pass@1.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" }, { "id": "automationbench", "name": "AutomationBench (DeepSeek V4.1 card)", "kind": "acc", "category": "llm", "description": "AutomationBench pass@1 with the official scaffold as labelled in the DeepSeek-V4.1-Flash card. Values differ from the 'AutomationBench (Public)' rows of the V4-Pro-0813 card, so kept separate from automationbench-public.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" }, { "id": "automationbench-public", "name": "AutomationBench (Public)", "kind": "acc", "category": "llm", "description": "AutomationBench public set as labelled in the DeepSeek V4-Pro-0813 / V4-Flash-0731 cards.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813" }, { "id": "agents-last-exam", "name": "Agents' Last Exam (pass@1, official scaffold)", "kind": "acc", "category": "llm", "description": "Agents' Last Exam pass@1 with the official scaffold as reported by the model publisher. Distinct from ale-openclaw-overall-pass (fixed OpenClaw + GUI-as-Tool harness).", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" }, { "id": "chartography-with-tools", "name": "Chartography with tools", "kind": "acc", "category": "llm", "description": "Chartography visual chart reasoning with tool use, pass@1; harness per note.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" }, { "id": "babyvision-with-tools", "name": "BabyVision with tools", "kind": "acc", "category": "llm", "description": "BabyVision visual reasoning with tool use (code interpreter/agent harness), pass@1.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" }, { "id": "zerobench-main-with-tools-pass5", "name": "ZeroBench main with tools (pass@5)", "kind": "acc", "category": "llm", "description": "ZeroBench main 100-question set with tool use, pass@5.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" }, { "id": "toolathlon-verified", "name": "Toolathlon-Verified", "kind": "acc", "category": "llm", "description": "Verified subset/release of Toolathlon (Tool Decathlon) as labelled by the source; separate from toolathlon.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813" }, { "id": "terminal-bench-1-terminus1", "name": "Terminal-Bench (pre-2.0) with Terminus 1", "kind": "acc", "category": "llm", "description": "Original Terminal-Bench (before 2.0) evaluated with the Terminus 1 framework, as in the DeepSeek-V3.1 card.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1" }, { "id": "frames", "name": "FRAMES", "kind": "acc", "category": "llm", "description": "Google FRAMES factuality/retrieval/reasoning benchmark accuracy as reported.", "url": "https://huggingface.co/google/frames-benchmark" }, { "id": "cnmo-2024", "name": "CNMO 2024", "kind": "acc", "category": "llm", "description": "Chinese National Mathematical Olympiad 2024 problems, pass@1.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1" }, { "id": "bfcl-v3-multiturn", "name": "BFCL v3 multi-turn", "kind": "acc", "category": "llm", "description": "Berkeley Function Calling Leaderboard v3 multi-turn category accuracy only (not the overall BFCL v3 score).", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528" }, { "id": "cluewsc", "name": "CLUEWSC", "kind": "acc", "category": "llm", "description": "CLUE Winograd Schema Challenge (Chinese) exact match.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1", "chance": 0.5 }, { "id": "arena-hard-v0-1-gpt4-1106", "name": "Arena-Hard (v0.1, GPT-4-1106 judge)", "kind": "acc", "category": "llm", "description": "Original Arena-Hard-Auto v0.1 win rate judged by GPT-4-1106-preview; not Arena-Hard v2.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1" }, { "id": "ifeval-prompt-strict", "name": "IFEval prompt-level strict", "kind": "acc", "category": "llm", "description": "IFEval prompt-level strict accuracy.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1" }, { "id": "tau-bench-airline-gpt41-user", "name": "τ-bench Airline (GPT-4.1 user)", "kind": "acc", "category": "llm", "description": "Original τ-bench Airline with GPT-4.1 simulating the user (DeepSeek R1-0528 card).", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528" }, { "id": "aime-2026-i", "name": "AIME 2026 I", "kind": "acc", "category": "llm", "description": "AIME 2026 contest I only (15 problems); distinct from aime-2026 (I and II).", "url": "https://huggingface.co/zai-org/GLM-5" }, { "id": "browsecomp-context-managed", "name": "BrowseComp with context management", "kind": "acc", "category": "llm", "description": "BrowseComp where the agent manages context (discard-all, reset, or folding strategy as stated in the note) beyond the model window. Separate from plain browsecomp.", "url": "https://huggingface.co/zai-org/GLM-5" }, { "id": "browsecomp-zh-context-managed", "name": "BrowseComp-ZH with context management", "kind": "acc", "category": "llm", "description": "BrowseComp-ZH with an agent context-management strategy (see note).", "url": "https://huggingface.co/stepfun-ai/Step-3.5-Flash" }, { "id": "critpt", "name": "CritPt", "kind": "acc", "category": "llm", "description": "CritPt research-level physics reasoning benchmark, accuracy as reported.", "url": "https://huggingface.co/zai-org/GLM-5.2" }, { "id": "gaia-text-only", "name": "GAIA (no-file subset)", "kind": "acc", "category": "llm", "description": "GAIA general-assistant benchmark restricted to questions without attached files, as labelled 'GAIA (no file)'.", "url": "https://huggingface.co/stepfun-ai/Step-3.5-Flash" }, { "id": "xbench-deepsearch-2505", "name": "xbench-DeepSearch (2025.05)", "kind": "acc", "category": "llm", "description": "xbench-DeepSearch May 2025 release, accuracy.", "url": "https://huggingface.co/stepfun-ai/Step-3.5-Flash" }, { "id": "xbench-deepsearch-2510", "name": "xbench-DeepSearch (2025.10)", "kind": "acc", "category": "llm", "description": "xbench-DeepSearch October 2025 release, accuracy.", "url": "https://huggingface.co/stepfun-ai/Step-3.5-Flash" }, { "id": "researchrubrics", "name": "ResearchRubrics", "kind": "acc", "category": "llm", "description": "ResearchRubrics deep-research rubric score as reported.", "url": "https://huggingface.co/stepfun-ai/Step-3.5-Flash" }, { "id": "forte", "name": "FORTE (AGI-Eval)", "kind": "acc", "category": "llm", "description": "FORTE general-agent benchmark (AGI-Eval-Official/FORTE) as reported.", "url": "https://huggingface.co/meituan-longcat/LongCat-2.0" }, { "id": "rwsearch", "name": "RWSearch", "kind": "acc", "category": "llm", "description": "RWSearch real-world search agent benchmark as reported.", "url": "https://huggingface.co/meituan-longcat/LongCat-2.0" }, { "id": "tau-bench-retail-gpt41-user", "name": "τ-bench Retail (GPT-4.1 user)", "kind": "acc", "category": "llm", "description": "Original τ-bench Retail with GPT-4.1 simulating the user.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528" }, { "id": "omni-math", "name": "Omni-MATH", "kind": "acc", "category": "llm", "description": "Omni-MATH olympiad-level mathematics (4,000+ problems) accuracy as reported.", "url": "https://huggingface.co/microsoft/Phi-4-reasoning" }, { "id": "livecodebench-20240801-20250201", "name": "LiveCodeBench (Aug 1, 2024–Feb 1, 2025)", "kind": "acc", "category": "llm", "description": "LiveCodeBench pass@1 on problems dated 2024-08-01 to 2025-02-01 as labelled by the source; version not stated.", "url": "https://huggingface.co/microsoft/Phi-4-reasoning" }, { "id": "flenqa-3k", "name": "FlenQA (3K-token subset)", "kind": "acc", "category": "llm", "description": "FlenQA long-input reasoning, 3K-token subset, accuracy.", "url": "https://huggingface.co/microsoft/Phi-4-reasoning", "chance": 0.5 }, { "id": "ifeval-strict-level-unspecified", "name": "IFEval strict (prompt/instruction level unspecified)", "kind": "acc", "category": "llm", "description": "IFEval strict accuracy where the source does not say prompt-level or instruction-level.", "url": "https://huggingface.co/microsoft/Phi-4-reasoning" }, { "id": "arena-hard-unspecified", "name": "Arena-Hard (version/judge unspecified)", "kind": "acc", "category": "llm", "description": "Arena-Hard win rate where the source does not state the version (v0.1 vs v2) or judge. Not merged with arena-hard-v2.", "url": "https://huggingface.co/microsoft/Phi-4-reasoning" }, { "id": "wildbench-v2", "name": "WildBench v2", "kind": "acc", "category": "llm", "description": "WildBench v2 score as reported by Mistral (percentage).", "url": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506" }, { "id": "gpqa-main", "name": "GPQA Main", "kind": "acc", "category": "llm", "description": "GPQA main set (448 questions), not Diamond; shots per note.", "url": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506", "chance": 0.25 }, { "id": "mbpp-plus-pass5", "name": "MBPP+ pass@5", "kind": "acc", "category": "llm", "description": "MBPP+ (EvalPlus) pass@5, not pass@1.", "url": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506" }, { "id": "humaneval-plus-pass5", "name": "HumanEval+ pass@5", "kind": "acc", "category": "llm", "description": "HumanEval+ (EvalPlus) pass@5, not pass@1.", "url": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506" }, { "id": "chartqa", "name": "ChartQA", "kind": "acc", "category": "llm", "description": "ChartQA test accuracy (relaxed accuracy as conventionally reported).", "url": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506" }, { "id": "docvqa", "name": "DocVQA", "kind": "acc", "category": "llm", "description": "DocVQA test ANLS as reported (0-100 scaled).", "url": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506" }, { "id": "deepsearchqa", "name": "DeepSearchQA", "kind": "acc", "category": "llm", "description": "DeepSearchQA deep-research question answering accuracy; harness per note.", "url": "https://huggingface.co/tencent/Hy3" }, { "id": "apex-agents", "name": "APEX-Agents (pass@1)", "kind": "acc", "category": "llm", "description": "APEX-Agents professional-task agent benchmark, pass@1 as reported.", "url": "https://huggingface.co/tencent/Hy3" }, { "id": "wildclawbench-35-text", "name": "WildClawBench (35 text-only queries)", "kind": "acc", "category": "llm", "description": "WildClawBench text-only subset of 35 queries with the OpenClaw harness (Tencent Hy3 appendix).", "url": "https://huggingface.co/tencent/Hy3" }, { "id": "skillsbench-79-text-claude-code", "name": "SkillsBench 79-task text-only (Claude Code, avg of 3)", "kind": "acc", "category": "llm", "description": "SkillsBench via Claude Code on 79 self-contained tasks excluding multimodal tasks, averaged over 3 runs. Distinct from skillsbench-87 and skillsbench-qwen-78.", "url": "https://huggingface.co/tencent/Hy3" }, { "id": "frontierscience-research", "name": "FrontierScience-Research", "kind": "acc", "category": "llm", "description": "OpenAI FrontierScience research-track score as reported by the source.", "url": "https://huggingface.co/tencent/Hy3" }, { "id": "frontierscience-olympiad", "name": "FrontierScience-Olympiad", "kind": "acc", "category": "llm", "description": "OpenAI FrontierScience olympiad-track accuracy; judge per note.", "url": "https://huggingface.co/tencent/Hy3" }, { "id": "usamo-2026", "name": "USAMO 2026", "kind": "acc", "category": "llm", "description": "USAMO 2026 proof problems score as reported (grading method per source).", "url": "https://huggingface.co/tencent/Hy3" }, { "id": "phybench", "name": "PHYBench", "kind": "acc", "category": "llm", "description": "PHYBench physics reasoning accuracy as reported.", "url": "https://huggingface.co/tencent/Hy3" }, { "id": "cmt-benchmark", "name": "CMT-Benchmark", "kind": "acc", "category": "llm", "description": "CMT-Benchmark (condensed-matter theory) accuracy as reported.", "url": "https://huggingface.co/tencent/Hy3" }, { "id": "superchem", "name": "SuperChem", "kind": "acc", "category": "llm", "description": "SuperChem chemistry reasoning accuracy as reported.", "url": "https://huggingface.co/tencent/Hy3" }, { "id": "cl-bench", "name": "CL-bench", "kind": "acc", "category": "llm", "description": "CL-bench context-learning benchmark score as reported.", "url": "https://huggingface.co/tencent/Hy3" }, { "id": "cl-bench-life", "name": "CL-bench life", "kind": "acc", "category": "llm", "description": "CL-bench life subset score as reported.", "url": "https://huggingface.co/tencent/Hy3" }, { "id": "hr-bench-4k", "name": "HR-Bench 4K", "kind": "acc", "category": "llm", "description": "HR-Bench high-resolution image understanding, 4K split, accuracy.", "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5", "chance": 0.25 }, { "id": "dailyomni", "name": "Daily-Omni", "kind": "acc", "category": "llm", "description": "Daily-Omni audio-visual video understanding accuracy.", "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5", "chance": 0.25 }, { "id": "videoholmes", "name": "Video-Holmes", "kind": "acc", "category": "llm", "description": "Video-Holmes complex video reasoning accuracy.", "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5", "chance": 0.1667 }, { "id": "facts-grounding", "name": "FACTS Grounding", "kind": "acc", "category": "llm", "description": "Google DeepMind FACTS Grounding factuality score as reported.", "url": "https://ai.google.dev/gemma/docs/core/model_card_3" }, { "id": "bird-sql-dev", "name": "BIRD-SQL (dev)", "kind": "acc", "category": "llm", "description": "BIRD text-to-SQL dev set execution accuracy as reported.", "url": "https://ai.google.dev/gemma/docs/core/model_card_3" }, { "id": "global-mmlu-lite", "name": "Global-MMLU-Lite", "kind": "acc", "category": "llm", "description": "Global-MMLU-Lite multilingual accuracy; distinct from full Global-MMLU.", "url": "https://ai.google.dev/gemma/docs/core/model_card_3", "chance": 0.25 }, { "id": "textvqa", "name": "TextVQA", "kind": "acc", "category": "llm", "description": "TextVQA accuracy (split as stated by the source).", "url": "https://ai.google.dev/gemma/docs/core/model_card_3" }, { "id": "librispeech-test-clean-wer", "name": "LibriSpeech test-clean WER", "kind": "err", "category": "stt", "description": "Word error rate on LibriSpeech test-clean (fraction). Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://www.openslr.org/12" }, { "id": "librispeech-test-other-wer", "name": "LibriSpeech test-other WER", "kind": "err", "category": "stt", "description": "Word error rate on LibriSpeech test-other (fraction). Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://www.openslr.org/12" }, { "id": "ami-wer", "name": "AMI WER (Open ASR / ESB test split)", "kind": "err", "category": "stt", "description": "Word error rate on the AMI meeting test split as packaged for the HF Open ASR Leaderboard (ESB). Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard" }, { "id": "earnings22-wer", "name": "Earnings-22 WER (short-form, Open ASR / ESB test split)", "kind": "err", "category": "stt", "description": "Word error rate on the segmented Earnings-22 test split used by the HF Open ASR Leaderboard (ESB); not the long-form recordings. Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard" }, { "id": "gigaspeech-wer", "name": "GigaSpeech test WER", "kind": "err", "category": "stt", "description": "Word error rate on GigaSpeech test (fraction). Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard" }, { "id": "spgispeech-wer", "name": "SPGISpeech test WER", "kind": "err", "category": "stt", "description": "Word error rate on the SPGISpeech test split used by the HF Open ASR Leaderboard (ESB). Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard" }, { "id": "tedlium-wer", "name": "TED-LIUM test WER (short-form)", "kind": "err", "category": "stt", "description": "Word error rate on the TED-LIUM (release 3) short-form test split as used by the HF Open ASR Leaderboard; release stated per result where the source gives it. Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard" }, { "id": "voxpopuli-en-wer", "name": "VoxPopuli English test WER", "kind": "err", "category": "stt", "description": "Word error rate on VoxPopuli English test (fraction). Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard" }, { "id": "commonvoice-9-en-wer", "name": "Common Voice 9.0 English test WER", "kind": "err", "category": "stt", "description": "Word error rate on Mozilla Common Voice 9.0 English test. Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/datasets/mozilla-foundation/common_voice_9_0" }, { "id": "commonvoice-en-wer-version-unspecified", "name": "Common Voice English WER (version unspecified)", "kind": "err", "category": "stt", "description": "Word error rate on a Common Voice English test set whose release the source does not state; kept apart from versioned Common Voice ids. Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://commonvoice.mozilla.org/" }, { "id": "fleurs-en-wer", "name": "FLEURS English test WER", "kind": "err", "category": "stt", "description": "Word error rate on FLEURS en_us test. Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/datasets/google/fleurs" }, { "id": "mls-en-wer", "name": "Multilingual LibriSpeech English test WER", "kind": "err", "category": "stt", "description": "Word error rate on MLS English test. Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://www.openslr.org/94" }, { "id": "fleurs-25eu-avg-wer", "name": "FLEURS 25 European languages, mean WER (NVIDIA set)", "kind": "err", "category": "stt", "description": "Unweighted mean WER on FLEURS test over the 25 European languages covered by NVIDIA Parakeet-TDT-0.6B-v3 / Canary-1B-v2 (bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk as enumerated in the Parakeet v3 card; the Canary card states only '25 Langs'). Punctuation and capitalisation removed before scoring.", "url": "https://huggingface.co/datasets/google/fleurs" }, { "id": "mls-6lang-avg-wer", "name": "Multilingual LibriSpeech 6 languages, mean WER (NVIDIA set)", "kind": "err", "category": "stt", "description": "Unweighted mean WER on MLS test over six non-English languages (es, fr, it, nl, pl, pt as enumerated in the Parakeet v3 card; the Canary card states only '6 Langs'). Punctuation and capitalisation removed.", "url": "https://www.openslr.org/94" }, { "id": "covost2-13lang-avg-wer", "name": "CoVoST 2 source speech, 13 languages, mean ASR WER (NVIDIA set)", "kind": "err", "category": "stt", "description": "Unweighted mean ASR WER on CoVoST 2 test source-language speech over 13 languages (de, en, es, et, fr, it, lv, nl, pt, ru, sl, sv, uk as enumerated in the Parakeet v3 card; the Canary card states only '13 Langs'). Transcription, not translation. Punctuation and capitalisation removed.", "url": "https://github.com/facebookresearch/covost" }, { "id": "tedlium-longform-wer", "name": "TED-LIUM long-form WER", "kind": "err", "category": "stt", "description": "Word error rate on full-length TED-LIUM long-form recordings (Open ASR long-form set), sequential or chunked decoding as stated per result. Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard" }, { "id": "meanwhile-longform-wer", "name": "Meanwhile long-form WER", "kind": "err", "category": "stt", "description": "Word error rate on the 'Meanwhile' long-form set (Open ASR long-form). Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard" }, { "id": "earnings21-longform-wer", "name": "Earnings-21 long-form WER", "kind": "err", "category": "stt", "description": "Word error rate on full Earnings-21 recordings (long-form). Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard" }, { "id": "earnings22-longform-wer", "name": "Earnings-22 long-form WER", "kind": "err", "category": "stt", "description": "Word error rate on full Earnings-22 recordings (long-form, not the segmented ESB split). Chunking/decoding per result. Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard" }, { "id": "rev16-longform-wer", "name": "Rev16 long-form WER", "kind": "err", "category": "stt", "description": "Word error rate on the Rev16 podcast long-form set. Lab-reported; text normalisation, segmentation and decoding follow the publishing card (usually the Whisper English normaliser used by the HF Open ASR Leaderboard) and are recorded per result. Separate evaluator keys per source; not the independent Open ASR Leaderboard snapshot.", "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard" }, { "id": "wenetspeech-test-net-err", "name": "WenetSpeech test_net error rate", "kind": "err", "category": "stt", "description": "Error rate on WenetSpeech test_net (Mandarin). The source labels its table WER; for Mandarin the unit is not further specified. Fraction.", "url": "https://github.com/wenet-e2e/WenetSpeech" }, { "id": "wenetspeech-test-meeting-err", "name": "WenetSpeech test_meeting error rate", "kind": "err", "category": "stt", "description": "Error rate on WenetSpeech test_meeting (Mandarin). The source labels its table WER; unit not further specified. Fraction.", "url": "https://github.com/wenet-e2e/WenetSpeech" }, { "id": "aishell2-test-err", "name": "AISHELL-2 test error rate", "kind": "err", "category": "stt", "description": "Error rate on AISHELL-2 test (Mandarin). The source labels its table WER; unit not further specified. Fraction.", "url": "https://www.aishelltech.com/aishell_2" }, { "id": "fleurs-zh-err", "name": "FLEURS Mandarin test error rate", "kind": "err", "category": "stt", "description": "Error rate on FLEURS cmn_hans_cn test. The source labels its table WER; unit not further specified. Fraction.", "url": "https://huggingface.co/datasets/google/fleurs" }, { "id": "commonvoice-zh-err-version-unspecified", "name": "Common Voice Mandarin error rate (version unspecified)", "kind": "err", "category": "stt", "description": "Error rate on a Common Voice zh-CN test set whose release the source does not state. Table labelled WER by the source. Fraction.", "url": "https://commonvoice.mozilla.org/" }, { "id": "seed-tts-en-wer-asr-unstated", "name": "Seed-TTS test-en WER (transcriber not stated)", "kind": "err", "category": "tts", "description": "Zero-shot voice cloning on Seed-TTS-Eval test-en, WER as a fraction, where the publisher does not state the ASR transcriber. Kept apart from seed-tts-en-whisper-v3-wer.", "url": "https://github.com/BytedanceSpeech/seed-tts-eval" }, { "id": "seed-tts-zh-err-asr-unstated", "name": "Seed-TTS test-zh error rate (transcriber not stated)", "kind": "err", "category": "tts", "description": "Zero-shot voice cloning on Seed-TTS-Eval test-zh, error rate as a fraction (source labels it WER), where the publisher does not state the ASR transcriber. Kept apart from seed-tts-zh-paraformer-cer.", "url": "https://github.com/BytedanceSpeech/seed-tts-eval" }, { "id": "gedit-bench-overall-lang-unspecified", "name": "GEdit-Bench overall (G_O, language split unspecified)", "kind": "acc", "category": "image", "description": "GEdit-Bench overall score G_O on the benchmark's 0–10 VLM-judge scale, divided by 10. The source does not state EN vs CN split or judge model; kept apart from GEditBench v2 Elo.", "url": "https://github.com/stepfun-ai/Step1X-Edit" }, { "id": "wan-bench-2-1-weighted", "name": "Wan-Bench (Wan2.1 release) weighted score", "kind": "acc", "category": "video", "description": "Alibaba Wan team's internal Wan-Bench used for the Wan2.1 release: 1,035 internal prompts, 14 major and 26 sub-dimensions, total computed as a weighted sum of dimension scores with weights derived from human preferences (per the Wan2.1 card). Published 0-1 weighted score. Internal benchmark: only the lab's own model rows are recorded.", "url": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B" }, { "id": "kris-bench-overall", "name": "KRIS-Bench overall", "kind": "acc", "category": "image", "description": "KRIS-Bench knowledge-based reasoning image-editing benchmark, published Overall (0–100) divided by 100; judge per benchmark protocol.", "url": "https://github.com/mercurystraw/Kris_Bench" }, { "id": "livebench-unspecified", "name": "LiveBench (release not stated)", "kind": "acc", "category": "llm", "description": "LiveBench overall average as reported by a model publisher without naming the LiveBench release; not merged with livebench-20241125.", "url": "https://arxiv.org/html/2412.08905v1" }, { "id": "bigcodebench-complete", "name": "BigCodeBench (Complete)", "kind": "acc", "category": "llm", "description": "BigCodeBench code-completion split pass@1 (full set unless stated); not the Instruct split.", "url": "https://arxiv.org/html/2503.01743v2" }, { "id": "bigcodebench-instruct", "name": "BigCodeBench (Instruct)", "kind": "acc", "category": "llm", "description": "BigCodeBench natural-language instruction split pass@1 (full set unless stated); not the Complete split.", "url": "https://arxiv.org/html/2503.01743v2" }, { "id": "bfcl-version-unspecified", "name": "BFCL (version not stated)", "kind": "acc", "category": "llm", "description": "Berkeley Function Calling Leaderboard score as reported by a publisher without naming v1/v2/v3/v4 or the aggregation; not merged with bfcl-v3 or bfcl-v4.", "url": "https://arxiv.org/html/2503.01743v2" }, { "id": "livecodebench-202408-202501", "name": "LiveCodeBench 2024-08 to 2025-01 (version not stated)", "kind": "acc", "category": "llm", "description": "LiveCodeBench pass@1 on problems dated August 2024 to January 2025 as stated by the source; release version not stated. Separate from all other windows.", "url": "https://arxiv.org/html/2504.21318v1" }, { "id": "wmt24pp-chrf", "name": "WMT24++ (chrF)", "kind": "acc", "category": "llm", "description": "WMT24++ translation scored with character-level F-score (chrF), divided by 100; not XCOMET.", "url": "https://arxiv.org/html/2503.19786v1" }, { "id": "vqav2-val", "name": "VQAv2 (val)", "kind": "acc", "category": "llm", "description": "VQAv2 validation-split VQA accuracy.", "url": "https://arxiv.org/html/2503.19786v1" }, { "id": "livecodebench-202407-202501", "name": "LiveCodeBench 2024-07 to 2025-01 (version not stated)", "kind": "acc", "category": "llm", "description": "LiveCodeBench pass@1 on problems dated 2024-07-01 to 2025-01-01 as stated by the source; release version not stated.", "url": "https://arxiv.org/html/2508.06471v1" }, { "id": "terminal-bench-pre-2", "name": "Terminal-Bench (pre-2.0 release)", "kind": "acc", "category": "llm", "description": "Terminal-Bench task success as reported before Terminal-Bench 2.0 existed (report dated before November 2025); task-set version not stated. Never merged with terminal-bench-2 or 2.1.", "url": "https://arxiv.org/html/2508.06471v1" }, { "id": "sysbench-isr", "name": "SysBench (ISR)", "kind": "acc", "category": "llm", "description": "SysBench system-message following, instruction satisfaction rate (ISR).", "url": "https://arxiv.org/html/2508.06471v1" }, { "id": "ifeval-strict-mean", "name": "IFEval strict, mean of prompt and instruction level", "kind": "acc", "category": "llm", "description": "Average of IFEval prompt-level strict and instruction-level strict accuracies (as defined by the Command A report); distinct from either component.", "url": "https://arxiv.org/pdf/2504.00698v2" }, { "id": "infobench", "name": "InFoBench", "kind": "acc", "category": "llm", "description": "InFoBench instruction-following (DRFR) score as reported.", "url": "https://arxiv.org/pdf/2504.00698v2" }, { "id": "livecodebench-v5-202410-202502", "name": "LiveCodeBench v5 (Oct 2024–Feb 2025)", "kind": "acc", "category": "llm", "description": "LiveCodeBench release v5 restricted to problems dated 10/2024–02/2025 as stated by the source; pass@1.", "url": "https://arxiv.org/pdf/2504.00698v2" }, { "id": "repoqa", "name": "RepoQA (32K, threshold 0.8)", "kind": "acc", "category": "llm", "description": "RepoQA needle-function search, 32K context, similarity threshold 0.8, match accuracy.", "url": "https://arxiv.org/pdf/2504.00698v2" }, { "id": "healthbench", "name": "HealthBench", "kind": "acc", "category": "llm", "description": "OpenAI HealthBench rubric-graded score (%), full set; not Hard or Consensus.", "url": "https://arxiv.org/html/2508.10925v1" }, { "id": "healthbench-hard", "name": "HealthBench Hard", "kind": "acc", "category": "llm", "description": "OpenAI HealthBench Hard subset rubric-graded score (%).", "url": "https://arxiv.org/html/2508.10925v1" }, { "id": "minif2f-pass1", "name": "miniF2F (pass@1)", "kind": "acc", "category": "llm", "description": "miniF2F formal theorem proving (Lean), pass@1 as reported; not pass@32.", "url": "https://arxiv.org/html/2512.20848v1" }, { "id": "ifbench-prompt-loose", "name": "IFBench prompt-level (loose)", "kind": "acc", "category": "llm", "description": "IFBench prompt-level loose accuracy; distinct from strict or unspecified IFBench scores.", "url": "https://arxiv.org/html/2606.15007v1" }, { "id": "bfcl-v2-live", "name": "BFCL v2 Live", "kind": "acc", "category": "llm", "description": "Berkeley Function Calling Leaderboard v2 'live' (user-contributed) subset accuracy; not the v3/v4 overall.", "url": "https://arxiv.org/html/2505.00949v5" }, { "id": "livecodebench-202408-202502", "name": "LiveCodeBench 2024-08 to 2025-02 (version not stated)", "kind": "acc", "category": "llm", "description": "LiveCodeBench pass@1 on problems dated 2024-08 to 2025-02 as stated by the source; release version not stated.", "url": "https://arxiv.org/html/2505.00949v5" }, { "id": "switchboard-wer", "name": "SwitchBoard WER", "kind": "err", "category": "stt", "description": "SwitchBoard (Hub5'00 SWB portion as used by the source) word error rate. Error-rate fraction as reported by the model publisher's own run; text normaliser, segmentation and decoding differ between publishers, so values are kept per source and are not the Hugging Face Open ASR Leaderboard numbers.", "url": "https://arxiv.org/html/2507.13264v1" }, { "id": "chime4-wer", "name": "CHiME-4 WER", "kind": "err", "category": "stt", "description": "CHiME-4 word error rate (subset as used by the source). Error-rate fraction as reported by the model publisher's own run; text normaliser, segmentation and decoding differ between publishers, so values are kept per source and are not the Hugging Face Open ASR Leaderboard numbers.", "url": "https://arxiv.org/html/2507.13264v1" }, { "id": "earnings21-10m-wer", "name": "Earnings-21 10-minute long-form WER", "kind": "err", "category": "stt", "description": "Earnings-21 long-form evaluation on 10-minute segments, word error rate. Error-rate fraction as reported by the model publisher's own run; text normaliser, segmentation and decoding differ between publishers, so values are kept per source and are not the Hugging Face Open ASR Leaderboard numbers.", "url": "https://arxiv.org/html/2507.13264v1" }, { "id": "earnings22-10m-wer", "name": "Earnings-22 10-minute long-form WER", "kind": "err", "category": "stt", "description": "Earnings-22 long-form evaluation on 10-minute segments, word error rate; not the short-form Open ASR split. Error-rate fraction as reported by the model publisher's own run; text normaliser, segmentation and decoding differ between publishers, so values are kept per source and are not the Hugging Face Open ASR Leaderboard numbers.", "url": "https://arxiv.org/html/2507.13264v1" }, { "id": "wise-overall-publisher-run", "name": "WISE overall (model publisher's run)", "kind": "acc", "category": "image", "description": "WISE world-knowledge text-to-image benchmark, overall WiScore (0-1) as run by the model's own publisher; judge model/version and prompt handling follow the publisher's report and may differ from the WISE authors' Table 1 runs (wise-wiscore-overall). Kept separate.", "url": "https://arxiv.org/html/2505.14683v3" }, { "id": "gedit-bench-en-overall", "name": "GEdit-Bench-EN overall (G_O, GPT-4.1 judge)", "kind": "acc", "category": "image", "description": "GEdit-Bench English full set, overall score G_O judged by GPT-4.1 per the official protocol, 0-10 scale divided by 10. Not GEdit-Bench v2 Elo.", "url": "https://arxiv.org/html/2505.14683v3" }, { "id": "imgedit-bench-overall", "name": "ImgEdit-Bench overall (GPT-4.1 judge)", "kind": "acc", "category": "image", "description": "ImgEdit-Bench overall (average over edit types), GPT-4.1 judge on a 1-5 scale, divided by 5 (so 0.2 is the scale floor).", "url": "https://arxiv.org/html/2506.03147v4" }, { "id": "tau3-banking", "name": "τ³-bench Banking", "kind": "acc", "category": "llm", "description": "Banking domain of τ³-bench (tau-cubed), task pass rate as reported by the source; user simulator and harness per note. Not the τ³-bench average.", "url": "https://huggingface.co/thinkingmachines/Inkling-Small/blob/8cc5877b44/README.md" }, { "id": "wildclawbench", "name": "WildClawBench", "kind": "acc", "category": "llm", "description": "WildClawBench agent benchmark, success rate as reported by the source.", "url": "https://huggingface.co/meta-models/Muse-Glimmer-30B" }, { "id": "gaia2", "name": "Gaia2", "kind": "acc", "category": "llm", "description": "Gaia2 (Meta Agents Research Environments) agent benchmark, overall success rate as reported.", "url": "https://huggingface.co/meta-models/Muse-Glimmer-30B" }, { "id": "skillsbench-with-skills", "name": "SkillsBench (with skills)", "kind": "acc", "category": "llm", "description": "SkillsBench with skills provided to the agent, as labelled by the source; task subset not stated. Distinct from skillsbench-87 and skillsbench-qwen-78.", "url": "https://huggingface.co/meta-models/Muse-Glimmer-30B" }, { "id": "mmau", "name": "MMAU", "kind": "acc", "category": "llm", "description": "MMAU massive multi-task audio understanding benchmark, accuracy (split per note).", "url": "https://huggingface.co/thinkingmachines/Inkling" }, { "id": "profbench", "name": "ProfBench", "kind": "acc", "category": "llm", "description": "ProfBench professional-domain report generation benchmark, score as reported by the source.", "url": "https://huggingface.co/ibm-granite/granite-4.2-30b" }, { "id": "birdbench", "name": "BIRD (text-to-SQL)", "kind": "acc", "category": "llm", "description": "BIRD-bench text-to-SQL execution accuracy as reported by the source ('BirdBench').", "url": "https://huggingface.co/ibm-granite/granite-4.2-30b" }, { "id": "gsm-symbolic", "name": "GSM-Symbolic", "kind": "acc", "category": "llm", "description": "GSM-Symbolic templated grade-school math accuracy; shots per note.", "url": "https://huggingface.co/ibm-granite/granite-4.1-8b" }, { "id": "deepmind-math", "name": "DeepMind Mathematics", "kind": "acc", "category": "llm", "description": "DeepMind Mathematics dataset accuracy as reported by the source (subset/aggregation as in the source's harness).", "url": "https://huggingface.co/ibm-granite/granite-4.1-8b" }, { "id": "livecodebench-v4", "name": "LiveCodeBench v4", "kind": "acc", "category": "llm", "description": "LiveCodeBench release v4 pass@1; distinct from v3, v5 and v6.", "url": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B" }, { "id": "mixeval-hard", "name": "MixEval-Hard", "kind": "acc", "category": "llm", "description": "MixEval-Hard ground-truth-based mixture benchmark score.", "url": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B" }, { "id": "omega", "name": "OMEGA (math generalisation)", "kind": "acc", "category": "llm", "description": "OMEGA out-of-distribution math reasoning benchmark accuracy as reported by Ai2.", "url": "https://huggingface.co/allenai/Olmo-3-7B-Think" }, { "id": "beyondaime", "name": "BeyondAIME", "kind": "acc", "category": "llm", "description": "BeyondAIME competition-math benchmark accuracy as reported by the source.", "url": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct" }, { "id": "korbench", "name": "KOR-Bench", "kind": "acc", "category": "llm", "description": "KOR-Bench knowledge-orthogonal reasoning accuracy.", "url": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct" }, { "id": "multi-swe-bench", "name": "Multi-SWE-bench (scaffold per note)", "kind": "acc", "category": "llm", "description": "Multi-SWE-bench multilingual issue resolution rate as reported by the source; language set/scaffold not stated. Distinct from multi-swe-bench-7lang-magentless.", "url": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct" }, { "id": "mgsm-5lang", "name": "MGSM, 5 languages (en, es, fr, ja, zh)", "kind": "acc", "category": "llm", "description": "MGSM restricted to English, Spanish, French, Japanese and Chinese (IBM Granite evaluation), 8-shot. Not full MGSM.", "url": "https://huggingface.co/ibm-granite/granite-4.1-8b" }, { "id": "supergpqa", "name": "SuperGPQA", "kind": "acc", "category": "llm", "description": "SuperGPQA graduate-level knowledge QA across 285 disciplines; accuracy as reported by the source. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md" }, { "id": "include", "name": "INCLUDE", "kind": "acc", "category": "llm", "description": "INCLUDE multilingual regional-knowledge exam benchmark (44 languages); accuracy as reported; language subset as stated by source. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md", "chance": 0.25 }, { "id": "mmlu-prox", "name": "MMLU-ProX", "kind": "acc", "category": "llm", "description": "MMLU-ProX multilingual MMLU-Pro; accuracy averaged over the languages the source reports (subset may differ by source). Not MMLU-ProX-Lite. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md" }, { "id": "mmlu-prox-lite", "name": "MMLU-ProX-Lite", "kind": "acc", "category": "llm", "description": "MMLU-ProX-Lite multilingual subset; accuracy as reported; language set per source. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md" }, { "id": "polymath", "name": "PolyMATH", "kind": "acc", "category": "llm", "description": "PolyMath multilingual mathematical reasoning; score as reported (difficulty-weighted accuracy where the source uses it). Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md" }, { "id": "livebench-20241125", "name": "LiveBench 2024-11-25", "kind": "acc", "category": "llm", "description": "LiveBench release 2024-11-25 average score. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md" }, { "id": "writingbench", "name": "WritingBench", "kind": "acc", "category": "llm", "description": "WritingBench generative-writing benchmark; judge-model score rescaled to 0-100 by the source. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md" }, { "id": "creative-writing-v3", "name": "Creative Writing v3", "kind": "acc", "category": "llm", "description": "EQ-Bench Creative Writing v3 as run and normalised by the model publisher; not the independent EQ-Bench leaderboard number. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md" }, { "id": "multichallenge", "name": "MultiChallenge", "kind": "acc", "category": "llm", "description": "MultiChallenge multi-turn conversation benchmark; judge-graded pass rate. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/blob/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md" }, { "id": "hmmt-nov-2025", "name": "HMMT November 2025", "kind": "acc", "category": "llm", "description": "November 2025 HMMT contest problems; not the February edition. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "hle-with-tools", "name": "Humanity's Last Exam with tools", "kind": "acc", "category": "llm", "description": "HLE with tool use (search/code) as allowed by the source's agent harness; separate from no-tool HLE. Subset (text-only vs full) as stated in note. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "browsecomp", "name": "BrowseComp", "kind": "acc", "category": "llm", "description": "OpenAI BrowseComp web-browsing agent benchmark; accuracy; harness/context management per source note. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/meituan-longcat/LongCat-2.0/blob/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md" }, { "id": "browsecomp-zh", "name": "BrowseComp-ZH", "kind": "acc", "category": "llm", "description": "Chinese BrowseComp-ZH browsing benchmark; accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "widesearch", "name": "WideSearch", "kind": "acc", "category": "llm", "description": "WideSearch broad information-seeking agent benchmark; item-level F1 or success as reported by source (see note). Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "seal-0", "name": "SealQA Seal-0", "kind": "acc", "category": "llm", "description": "Seal-0 subset of SealQA search-augmented QA; accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "vita-bench", "name": "VitaBench", "kind": "acc", "category": "llm", "description": "VitaBench interactive agent tasks (Meituan); average success rate as reported. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "deepplanning", "name": "DeepPlanning", "kind": "acc", "category": "llm", "description": "DeepPlanning long-horizon planning agent benchmark (Qwen); score as reported. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "ojbench", "name": "OJBench", "kind": "acc", "category": "llm", "description": "OJBench competition-level code generation (online-judge problems); pass rate. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/6cbffae6d8e28b986a6b17bd36f42f9fa0f1f0a5/README.md" }, { "id": "fullstackbench-en", "name": "FullStackBench (English)", "kind": "acc", "category": "llm", "description": "FullStackBench English split; pass rate. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "fullstackbench-zh", "name": "FullStackBench (Chinese)", "kind": "acc", "category": "llm", "description": "FullStackBench Chinese split; pass rate. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "nova-63", "name": "NOVA-63", "kind": "acc", "category": "llm", "description": "NOVA-63 multilingual benchmark (63 languages) as reported by Qwen; score as reported. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "global-piqa", "name": "Global PIQA", "kind": "acc", "category": "llm", "description": "Global PIQA multilingual physical commonsense; accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md", "chance": 0.5 }, { "id": "wmt24pp", "name": "WMT24++", "kind": "acc", "category": "llm", "description": "WMT24++ machine translation (55 languages); metric as reported by source (see note; typically XCOMET-XXL). Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "maxife", "name": "MAXIFE", "kind": "acc", "category": "llm", "description": "MAXIFE multilingual instruction-following benchmark; accuracy as reported. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "mmmu", "name": "MMMU (val)", "kind": "acc", "category": "llm", "description": "MMMU validation accuracy; not MMMU-Pro. Link: first pinned publisher document in this batch that reports it. Chance 0.221: published random baseline (https://arxiv.org/abs/2311.16502; MMMU paper, main results table: 'Random Choice 22.1' (Validation Overall, 900) and 'Frequent Choice 26.8'. Text: 'For reference, we add Random Choice and Freque).", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md", "chance": 0.221 }, { "id": "mathvista-mini", "name": "MathVista (testmini)", "kind": "acc", "category": "llm", "description": "MathVista testmini accuracy. Link: first pinned publisher document in this batch that reports it. Chance 0.179: published random baseline (https://arxiv.org/abs/2310.02255; MathVista (ICLR 2024), Table 2 (testmini), 'Heuristics baselines': 'Random chance - 17.9' (ALL). Text: 'We include random chance (i.e., one of the options in mu).", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md", "chance": 0.179 }, { "id": "dynamath", "name": "DynaMath", "kind": "acc", "category": "llm", "description": "DynaMath dynamic visual math; worst-case or average accuracy as the source states. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "realworldqa", "name": "RealWorldQA", "kind": "acc", "category": "llm", "description": "RealWorldQA accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "mmstar", "name": "MMStar", "kind": "acc", "category": "llm", "description": "MMStar accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md", "chance": 0.25 }, { "id": "mmbench-en-dev-v1-1", "name": "MMBench-EN dev v1.1", "kind": "acc", "category": "llm", "description": "MMBench English dev v1.1 accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "simplevqa", "name": "SimpleVQA", "kind": "acc", "category": "llm", "description": "SimpleVQA factual visual QA accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "hallusionbench", "name": "HallusionBench", "kind": "acc", "category": "llm", "description": "HallusionBench average accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "charxiv-rq", "name": "CharXiv (reasoning questions)", "kind": "acc", "category": "llm", "description": "CharXiv reasoning-question accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "ai2d-test", "name": "AI2D test", "kind": "acc", "category": "llm", "description": "AI2D test accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md", "chance": 0.25 }, { "id": "ocrbench", "name": "OCRBench", "kind": "acc", "category": "llm", "description": "OCRBench score normalised to 0-100 by the source (raw 0-1000 divided by 10). Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "cc-ocr", "name": "CC-OCR", "kind": "acc", "category": "llm", "description": "CC-OCR overall score. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "mmlongbench-doc", "name": "MMLongBench-Doc", "kind": "acc", "category": "llm", "description": "MMLongBench-Doc accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "omnidocbench-1-5-overall", "name": "OmniDocBench 1.5 overall (higher is better)", "kind": "acc", "category": "llm", "description": "OmniDocBench v1.5 overall score as reported (higher is better). Distinct from omnidocbench-1-5 (edit distance, lower is better). Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "videomme-with-sub", "name": "Video-MME (with subtitles)", "kind": "acc", "category": "llm", "description": "Video-MME accuracy with subtitles. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md", "chance": 0.25 }, { "id": "videomme-without-sub", "name": "Video-MME (without subtitles)", "kind": "acc", "category": "llm", "description": "Video-MME accuracy without subtitles. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md", "chance": 0.25 }, { "id": "videommmu", "name": "VideoMMMU", "kind": "acc", "category": "llm", "description": "VideoMMMU accuracy. Link: first pinned publisher document in this batch that reports it. Chance 0.14: published random baseline (https://arxiv.org/abs/2501.13826; Video-MMMU paper, main results table: 'Random Choice 14.00' (Overall; Perception 12.00, Comprehension 14.00, Adaptation 16.00).).", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md", "chance": 0.14 }, { "id": "mlvu", "name": "MLVU", "kind": "acc", "category": "llm", "description": "MLVU long-video understanding accuracy (M-Avg as reported). Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "mvbench", "name": "MVBench", "kind": "acc", "category": "llm", "description": "MVBench accuracy. Link: first pinned publisher document in this batch that reports it. Chance 0.273: published random baseline (https://arxiv.org/abs/2311.17005; MVBench paper, Table 2: 'Random - 27.3' (Avg column; per-task 20.0-33.3). Caption: '“Random” refers to results from random guesses.').", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md", "chance": 0.273 }, { "id": "lvbench", "name": "LVBench", "kind": "acc", "category": "llm", "description": "LVBench long-video accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md", "chance": 0.25 }, { "id": "mmvu", "name": "MMVU", "kind": "acc", "category": "llm", "description": "MMVU expert-level video understanding accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "screenspot-pro", "name": "ScreenSpot-Pro", "kind": "acc", "category": "llm", "description": "ScreenSpot-Pro GUI grounding accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "osworld-verified", "name": "OSWorld-Verified", "kind": "acc", "category": "llm", "description": "OSWorld-Verified computer-use success rate. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "androidworld", "name": "AndroidWorld", "kind": "acc", "category": "llm", "description": "AndroidWorld mobile-agent success rate. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "erqa", "name": "ERQA", "kind": "acc", "category": "llm", "description": "ERQA embodied reasoning QA accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "countbench", "name": "CountBench", "kind": "acc", "category": "llm", "description": "CountBench object-counting accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "embspatialbench", "name": "EmbSpatial-Bench", "kind": "acc", "category": "llm", "description": "EmbSpatial-Bench accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "refspatialbench", "name": "RefSpatial-Bench", "kind": "acc", "category": "llm", "description": "RefSpatial-Bench accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "slake", "name": "SLAKE", "kind": "acc", "category": "llm", "description": "SLAKE medical VQA accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "pmc-vqa", "name": "PMC-VQA", "kind": "acc", "category": "llm", "description": "PMC-VQA medical VQA accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "zerobench-sub", "name": "ZEROBench subquestions", "kind": "acc", "category": "llm", "description": "ZEROBench sub-question accuracy (not the main 100-question pass@1). Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "vlmsareblind", "name": "VLMs Are Blind", "kind": "acc", "category": "llm", "description": "VLMs Are Blind low-level visual tasks accuracy. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "lingoqa", "name": "LingoQA", "kind": "acc", "category": "llm", "description": "LingoQA driving VQA score as reported. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B/blob/dc4d348443bc740c68e2d77492492c11606384d5/README.md" }, { "id": "we-math", "name": "We-Math", "kind": "acc", "category": "llm", "description": "We-Math visual math score as reported. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md" }, { "id": "multipl-e", "name": "MultiPL-E", "kind": "acc", "category": "llm", "description": "MultiPL-E multilingual code generation pass@1 averaged over the languages the source reports. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/cdbee75f17c01a7cc42f958dc650907174af0554/README.md" }, { "id": "nl2repo", "name": "NL2Repo", "kind": "acc", "category": "llm", "description": "NL2Repo repository-level generation benchmark; score as reported. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md" }, { "id": "mcp-atlas", "name": "MCP-Atlas (public set)", "kind": "acc", "category": "llm", "description": "MCP-Atlas public set pass rate; harness/judge per note. Separate from mcp-atlas-v1-500 when the source does not state v1 public500. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md" }, { "id": "toolathlon", "name": "Toolathlon (Tool Decathlon)", "kind": "acc", "category": "llm", "description": "Toolathlon / Tool Decathlon agent benchmark pass@1 as reported. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/README.md" }, { "id": "cybergym", "name": "CyberGym", "kind": "acc", "category": "llm", "description": "CyberGym vulnerability-reproduction agent benchmark; success rate. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/zai-org/GLM-5.1/blob/26e1bd6e011feb778d25ae34b09b07074139d92d/README.md" }, { "id": "tau3-bench", "name": "τ³-bench", "kind": "acc", "category": "llm", "description": "τ³-bench (tau-cubed) agent benchmark average as reported; distinct from τ²-bench. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/995ad96eacd98c81ed38be0c5b274b04031597b0/README.md" }, { "id": "ruler-128k", "name": "RULER 128k", "kind": "acc", "category": "llm", "description": "RULER long-context score at 128k tokens. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct/blob/497f1dca95ebdec98e41d517b9f060ee753c902f/README.md" }, { "id": "ruler-64k", "name": "RULER 64k", "kind": "acc", "category": "llm", "description": "RULER long-context score at 64k tokens. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md" }, { "id": "cruxeval-o", "name": "CRUXEval-O", "kind": "acc", "category": "llm", "description": "CRUXEval output prediction pass@1. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md" }, { "id": "bigcodebench", "name": "BigCodeBench", "kind": "acc", "category": "llm", "description": "BigCodeBench pass@1; split (full/hard, complete/instruct) per note. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md" }, { "id": "drop", "name": "DROP", "kind": "acc", "category": "llm", "description": "DROP reading comprehension F1 as reported. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad/README.md" }, { "id": "minerva-math", "name": "Minerva MATH", "kind": "acc", "category": "llm", "description": "MATH scored with Minerva-style answer normalisation, as reported. Link: first pinned publisher document in this batch that reports it.", "url": "https://huggingface.co/ibm-granite/granite-4.0-h-small/blob/b8c0982bab7fde4eb48110f5a069527c008fab39/README.md" }, { "id": "aime-2025-with-tools", "name": "AIME 2025 with tools", "kind": "acc", "category": "llm", "description": "AIME 2025 with tool-integrated reasoning (code execution) as labelled 'w/ Tools' by the source; separate from no-tool aime-2025. Tool setup per the source's TIR guide and the note.", "url": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md" }, { "id": "gpqa-unspecified-with-tools", "name": "GPQA (subset unspecified) with tools", "kind": "acc", "category": "llm", "description": "GPQA with tool-integrated reasoning as labelled 'w/ Tools' by the source. The source labels only 'GPQA'; Diamond identity is not inferred. Separate from all no-tool GPQA ids.", "url": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md", "chance": 0.25 }, { "id": "livecodebench-v6-window-unspecified-with-tools", "name": "LiveCodeBench v6 (window unspecified) with tools", "kind": "acc", "category": "llm", "description": "LiveCodeBench v6 pass@1 with tool-integrated reasoning as labelled 'w/ Tools' by the source; date window not stated. Separate from no-tool livecodebench-v6-window-unspecified.", "url": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md" }, { "id": "tau2-bench-weighted", "name": "τ²-bench overall (weighted)", "kind": "acc", "category": "llm", "description": "τ²-bench overall score labelled '(Weighted)' by the source: domain results combined with a weighting the source does not spell out. Kept separate from tau2-bench (simple average of Airline, Retail and Telecom).", "url": "https://huggingface.co/zai-org/GLM-4.6/blob/be72194883d968d7923a07e2f61681ea9a2826d1/README.md" }, { "id": "agents-last-exam-unspecified", "name": "Agents' Last Exam (metric/harness unspecified)", "kind": "acc", "category": "llm", "description": "Agents' Last Exam score where the publisher states neither the metric nor the harness (the GLM-5.3-Flash card's footnote for it is blank). Not merged with agents-last-exam (pass@1, official scaffold) or ale-openclaw-overall-pass.", "url": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/eb9eb208eb0d988989d07a6a12d0fdeb5f52574a/README.md" }, { "id": "automationbench-v1-0-6", "name": "AutomationBench v1.0.6", "kind": "acc", "category": "llm", "description": "AutomationBench (zapier/AutomationBench) v1.0.6 including the null-type handling fix from PR #13, as stated by the source. Not merged with automationbench (DeepSeek V4.1 card label) or automationbench-public, whose versions are not stated.", "url": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/eb9eb208eb0d988989d07a6a12d0fdeb5f52574a/README.md" }, { "id": "collie", "name": "COLLIE (constrained text generation)", "kind": "acc", "category": "llm", "description": "COLLIE constrained text generation benchmark (Yao et al., 2023, arXiv:2307.08689); constraint-satisfaction score as reported by the source. Subset and aggregation not stated by the source.", "url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md" }, { "id": "simplevqa-with-tools", "name": "SimpleVQA with tools (search)", "kind": "acc", "category": "llm", "description": "SimpleVQA factual visual QA with tool use (search) allowed, labelled 'SimpleVQA (with Tool)' in the chart and 'SimpleVQA (Search)' in the prose of the source. Separate from no-tool simplevqa.", "url": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/blob/5f6244077ac62e04eec3f320501ff8c2b293373a/README.md" }, { "id": "vstar-with-python", "name": "V* Bench with Python tool", "kind": "acc", "category": "llm", "description": "V* Bench visual search (Wu & Xie, 2023) accuracy with a Python/code tool, labelled 'V* (with Python)' by the source.", "url": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/blob/5f6244077ac62e04eec3f320501ff8c2b293373a/README.md" }, { "id": "claw-eval-1-1-unspecified", "name": "ClawEval 1.1 (metric unspecified)", "kind": "acc", "category": "llm", "description": "ClawEval-1.1 (2026-05-09 release) where the source does not state the metric (Pass^3 vs average). Not merged with claw-eval-pass3 or claw-eval-avg.", "url": "https://huggingface.co/stepfun-ai/Step-3.7-Flash/blob/5f6244077ac62e04eec3f320501ff8c2b293373a/README.md" }, { "id": "afrispeech-multibench-accents7-mean-wer", "name": "AfriSpeech-MultiBench · 7-region African-accent mean WER", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2511.14255v1", "metric": "word_error_rate", "setup": "AfriSpeech-MultiBench (Intron Health): African-accented English speech; Table 6 reports WER per accent region (Nigeria, Ghana, Kenya, Rwanda, Uganda, South Africa, North Africa) and their unweighted mean. Zero-shot, default decoding, 16 kHz mono, filler removal and number normalisation (§3.3).", "score_unit": "error ratio (source % / 100)", "caveats": [ "Evaluator (Intron) sells a competing ASR (Sahara), not in this series.", "Evaluated autumn 2025; hosted APIs may have changed since.", "Some rows in the paper contain arithmetic slips (Table 4 averages; Table 6 Chirp V3 printed mean 23.67 vs 23.39 from its cells). The build keeps only Table 6 rows whose printed mean matches their cells, so Chirp 3 is excluded from this series." ] }, { "id": "afrispeech-multibench-parliament-wer", "name": "AfriSpeech-MultiBench · Parliament (legal domain) WER", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2511.14255v1", "metric": "word_error_rate", "setup": "AfriSpeech-MultiBench 'Parl' subset: African-accented parliamentary speech (8,068 clips, 35.9 h, 4 countries; Table 1). Zero-shot runs by Intron Health, Table 3.", "score_unit": "error ratio (source % / 100)", "caveats": [ "gpt-4o-transcribe scores 64.4% here (outlier, likely long-audio handling); kept as published.", "Same runs/paper as afrispeech-multibench-accents7-mean-wer (correlated series)." ] }, { "id": "oual-arabic-6set-avg-wer", "name": "Open Universal Arabic ASR Leaderboard · 6-test-set average WER", "category": "stt", "kind": "err", "url": "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py", "metric": "word_error_rate", "setup": "Open Universal Arabic ASR Leaderboard protocol (Elm Research Center, arXiv:2412.13788; evaluation scripts in GitHub Natural-Language-Processing-Elm/open_universal_arabic_asr_leaderboard): multi-dialect Arabic test sets SADA (10 dialects, 10.7 h), Common Voice 18.0 Arabic (12.6 h), MASC clean-test (10.5 h) and noisy-test (14.9 h), MGB-2 (9.6 h) and Casablanca (8 dialects, 7.7 h). Published 'Average WER' = unweighted mean of the six per-set WERs (verified by script). Each evaluator (Elm original board; the msalhab96 Extended Edition fork using the same test sets and scripts) is its own series; per-set WERs kept as components.", "score_unit": "error ratio (source % / 100)", "setup_sources": [ "https://huggingface.co/spaces/elmresearchcenter/open_universal_arabic_asr_leaderboard/blob/667db1e1e40c38f1c5667f7dca1a90eac113231d/app.py", "https://huggingface.co/spaces/msalhab96/open_universal_arabic_asr_leaderboard_all/blob/7c2c457af04fb9e6846baf04121f899dd99dd193/app.py" ], "caveats": [ "Unweighted mean over six sets of very different difficulty (Casablanca and SADA dominate); dialectal Arabic WER is high for all systems.", "Normalization and decoding settings are defined by the Elm evaluation scripts, not restated per row; hosted APIs have no immutable build id.", "Common Voice 18.0 and MGB-2 are public; some systems may have trained on them." ] }, { "id": "betthupferl-de-standard-audio-wer", "name": "Betthupferl (German dialect ASR benchmark) · Standard German audio → Standard German transcript WER", "category": "stt", "kind": "err", "url": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv", "metric": "word_error_rate", "setup": "Bavarian/German-dialect ASR Benchmark (Hugging Face Space kamaal-08/bavarian-asr-leaderboard) on the complete Betthupferl evaluation set. Task Standard→Standard: the 531 Standard German recordings scored against their Standard German transcripts. Mean WER ('metric_basis: mean'), validated full-split runs by the board maintainer (paper reproductions of the Betthupferl authors' setup plus additional benchmarks). Values published as fractions.", "score_unit": "error ratio (source fraction)", "sample_count": 531, "caveats": [ "Small set (531 utterances) of children's bedtime-story style read speech.", "metric_basis 'mean' (average of per-recording WER, not pooled corpus WER).", "Board maintainer is an independent Hugging Face user; decoding settings per model are not listed in the CSV." ] }, { "id": "betthupferl-de-dialect-to-standard-wer", "name": "Betthupferl (German dialect ASR benchmark) · dialect audio → Standard German transcript WER", "category": "stt", "kind": "err", "url": "https://huggingface.co/spaces/kamaal-08/bavarian-asr-leaderboard/blob/1a8d0ac6d092ce3d5fc46d41e9cbc2cd74240bed/data/leaderboard.csv", "metric": "word_error_rate", "setup": "Same board and runs. Task Dialect→Standard: 3,257 German dialect recordings (Bavarian and other regional dialects) scored against the normalized Standard German transcript, i.e. dialect-to-standard transcription/translation. Mean WER ('metric_basis: mean'). The Dialect→Dialect task (dialect-preserving references, WER ~0.61-0.63 for all four catalogue models) is not used because it barely separates systems.", "score_unit": "error ratio (source fraction)", "sample_count": 3257, "caveats": [ "Dialect speech with Standard German references: errors include legitimate dialect-to-standard wording differences.", "metric_basis 'mean' (average of per-recording WER)." ] }, { "id": "botnoi-thai-telephony-cer", "name": "BOTNOI Thai telephony · CER", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2608.24916v1", "metric": "character_error_rate", "setup": "BOTNOI in-house Thai telephony evaluation set (real voicebot calls and prompted handset recordings). Zero-shot baselines scored with CER by Botnoi Group (Table 3).", "score_unit": "error ratio (source % / 100)", "caveats": [ "Private in-house test set; not reproducible.", "Evaluator sells the competing fine-tuned Canary system (not in this series)." ] }, { "id": "botnoi-thai-fleurs-cer", "name": "BOTNOI report · FLEURS Thai CER", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2608.24916v1", "metric": "character_error_rate", "setup": "FLEURS Thai test set, CER, zero-shot baselines run by Botnoi Group (Table 2; Common Voice 23 Thai CER kept as a component).", "score_unit": "error ratio (source % / 100)", "caveats": [ "Only three catalogue models; normalisation details not stated." ] }, { "id": "cl-spanish-asr-3set-avg-wer", "name": "Open ASR Leaderboard – Chilean Spanish · 3-dataset average WER", "category": "stt", "kind": "err", "url": "https://huggingface.co/spaces/idsudd/open_asr_leaderboard_cl/blob/7dfb852bee82c9b602548abb33213449c360e31b/results.csv", "metric": "word_error_rate", "setup": "Chilean Spanish adaptation of the Hugging Face Open ASR Leaderboard evaluation framework (Space idsudd/open_asr_leaderboard_cl, Alonso Astroza), test data astroza/es-cl-asr-test-only: Google Chilean Spanish (4,374 utterances, read sentences), Common Voice Chilean-filtered (152) and Datarisas (50, spontaneous stand-up comedy from Festival de Viña). Zero-shot normalised WER with a multilingual normalizer that keeps Spanish accents; runs 2025-10-26/27 (Transformers, NeMo, Voxtral backends; job ids in results.csv). 'Average WER' = unweighted mean of the three per-dataset WERs, exactly as the Space's load_results() computes it; per-dataset WERs kept as components.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Unweighted 3-set mean: the 50-utterance Datarisas set (WER 16-35%) carries a third of the weight.", "Single run per model; Google Chilean Spanish and Common Voice are public and may be in training data." ] }, { "id": "megaasr-adverse3-mean-wer", "name": "Mega-ASR adverse-condition suite · mean WER (CHiME-4, VOiCES, NOIZEUS)", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2605.19833v1", "metric": "word_error_rate", "setup": "Unweighted mean of three adverse-condition English ASR averages as published in Table 2 'Avg.': CHiME-4 (real+simulated noisy), VOiCES (far-field rooms rm1-rm4) and NOIZEUS (0/5/10/15 dB SNR). Baselines run by the Mega-ASR authors (NTU/NUS/Shanghai AI Lab).", "score_unit": "error ratio (source % / 100)", "caveats": [ "Authors' own runs inferred (no public source reports these per-condition cells for these baselines); decoding settings not stated.", "gpt-4o-transcribe identity rests on the abbreviated label 'GPT-4o-trans.'.", "NOIZEUS is small (30 IEEE sentences under several noise types); its 0 dB condition dominates the NOIZEUS mean." ] }, { "id": "openbench-earnings22-kws-nokw-wer", "name": "Argmax OpenBench · earnings22-keywords WER (no keyword prompting)", "category": "stt", "kind": "err", "url": "https://github.com/argmaxinc/OpenBench/blob/e13108070b9693f2646e4a3e22e4b2d0130d779f/BENCHMARKS.md", "metric": "word_error_rate", "setup": "Argmax OpenBench Keyword Recognition benchmark, 'no keywords' baseline column: plain transcription WER on argmaxinc/earnings22-kws-golden test split (Earnings-22 corporate earnings calls re-annotated by Argmax, 15-second chunks each containing at least one annotated name/keyword), no custom vocabulary or keyword prompting. Cloud APIs via vendor SDKs (pre-recorded); open models on Apple Silicon (openai-whisper; Argmax WhisperKit Pro SDK for Parakeet and Qwen3-ASR). Runs 2025-12-12 to 2026-08-04.", "caveats": [ "Argmax SDK builds of open models (compressed Parakeet V3/V2, Qwen3-ASR) are excluded; the evaluator sells that SDK (vendor-run comparison).", "Parakeet v2 and ElevenLabs rows have no printed run date or model version; identities rest on the row label and on the pipeline default (scribe_v2) committed with the ElevenLabs row.", "Rows were run on different dates (2025-12 to 2026-08) and hardware.", "Earnings-22 chunks selected for containing named entities; harder than generic earnings-call audio." ] }, { "id": "persian-triple-threat-2split-wer", "name": "Persian ASR Triple Threat · VisualEars6669 + FLEURS-fa equal-weight WER", "category": "stt", "kind": "err", "url": "https://huggingface.co/spaces/Reza2kn/PersianASR-TrippleThreat/blob/63e7e3f8b3754a92d6c0ac2a9db84344cf7e49bb/results.json", "metric": "word_error_rate", "setup": "Persian ASR Triple Threat Leaderboard (Reza2kn): VisualEars6669 (6,669 challenging Persian clips, 10.49 h, clean/farfield/obstructed conditions) and the full FLEURS-fa set (4,341 rows, raw_transcription references). WER under the board's uniform 'compound-fair' normalization (punctuation, spaces, half-spaces/ZWNJ, diacritics ignored; compound and homophone folding), applied to every model. Score = equal-weight mean of the two per-split WERs, i.e. the board's own WER pillar ('each metric pillar is averaged 50/50 across VisualEars6669 and FLEURS-fa Full'); per-split WERs kept as components. The board's headline Triple Threat composite (60% S3 + 20% WER + 20% CER) is not used.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Maintainer builds competing Persian ASR models (Shenava); those rows are not used.", "Rows run through the OpenRouter gateway on this board (e.g. chirp-3, gpt-transcribe, nova-3, mai-transcribe-1.5) are excluded; only direct-API and open-weight runs are used.", "VisualEars6669 is the maintainer's curated set; the fair normalization is the maintainer's own." ] }, { "id": "peste-fleurs-fa-cer", "name": "PESTE · Persian FLEURS (fa_ir) corpus CER", "category": "stt", "kind": "err", "url": "https://github.com/ArmanJR/PESTE-Benchmark/blob/4ca05cd34f90342d76d12d9e8074e7a26de97e96/generated/leaderboard.csv", "metric": "character_error_rate", "setup": "PESTE v2.1 suite fleurs-fa-ir-v1: all 871 Google FLEURS Persian (fa_ir) test recordings, pinned audio manifest; immutable fa-v1 normalization of references and predictions; corpus CER (source's primary metric, ignores normalized whitespace; WER kept in notes) with 10,000-replicate utterance bootstrap CI. Offline execution on one RTX 6000 Ada in a digest-pinned GHCR runtime image; checkpoint-native precision; pinned HF repository revisions (models/.json).", "sample_count": 871, "caveats": [ "FLEURS is public read speech and may overlap model training data (source caveat).", "CER is primary because Persian WER is sensitive to word segmentation; WER is recorded in each note.", "Most leaderboard rows are Persian fine-tunes not in the catalogue; only three catalogue models are covered." ] }, { "id": "quranic-asr-heldout600-wer", "name": "Quranic ASR Leaderboard · held-out recitation (600 clips) overall WER", "category": "stt", "kind": "err", "url": "https://huggingface.co/spaces/Quran-Lab/quranic-asr-leaderboard/blob/47d9dd9840ed6897c2c305e4319d16f13360dea0/README.md", "metric": "word_error_rate", "setup": "Quran-Lab leakage-free Quranic recitation benchmark v1.1 (Quran-Lab/quranic-asr-benchmark): 600 clips, 200 each from EveryAyah held-out reciters (studio), QUL Al-Nufais (canonical mushaf references) and phone-recorded recitation. Arabic language hint on every system, no biasing or tuning, same official score.py normalisation (diacritic/Quranic-sign strip, alef/ya/ta-marbuta folding). Score = overall WER on all 600 clips as published.", "score_unit": "error ratio (source % / 100)", "sample_count": 600, "caveats": [ "Highly specialised domain (Classical Arabic recitation); general-purpose ASR ranks differ from conversational benchmarks.", "Evaluator also builds competing Quranic models (not used here).", "Cloud API runs from 2026-07; hosted models may have changed." ] }, { "id": "shmyrev-ru-11set-avg-wer", "name": "Russian ASR Leaderboard (N. Shmyrev test sets) · 11-dataset mean WER", "category": "stt", "kind": "err", "url": "https://huggingface.co/spaces/ESpeech/russian_asr_leaderboard/blob/cc62862c6f7d12840458f868a17a3058da7e639f/index.html", "metric": "word_error_rate", "setup": "Russian ASR benchmark by Nikolay Shmyrev, test date 07.05.2026, published as the static ESpeech Russian ASR Leaderboard. 'Среднее' = mean WER over 11 Russian test sets: audiobooks (Аудиокниги АЦ), Russian LibriSpeech, Common Voice 12.0, Golos Crowd, Golos Farfield, Sova device, TV broadcast (Телевещание), medicine (Медицина), Yandex commands, phone calls orders and phone calls support. Unweighted mean of the per-set WERs (verified by script). Noise and rare-word stress tests are excluded (not filled for all models).", "score_unit": "error ratio (source % / 100)", "caveats": [ "Engine labels are human-readable names, not Hugging Face ids; decoding settings are not stated per row.", "Several sets (Golos, Common Voice, Russian LibriSpeech) are public and may overlap training data of Russian-specialised systems.", "Board authors also build Russian ASR systems (Vosk; ESpeech), none of which are catalogue models." ] }, { "id": "sap-dysarthric-rated11k-zeroshot-wer", "name": "Speech Accessibility Project dysarthric speech · zero-shot WER (rated subset, N=11,218)", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2605.02782v2", "metric": "word_error_rate", "sample_count": 11218, "setup": "Speech Accessibility Project (SAP) dysarthric English speech, rated subset of 11,218 utterances from speakers with Parkinson's disease, ALS, Down syndrome and cerebral palsy. Zero-shot, audio-only inference; SAP challenge scoring (Whisper EnglishTextNormalizer, minimum over the two reference variants, per-utterance WER clipped at 1.0). Run by University of Zurich / ETH Zurich authors (Table 4).", "score_unit": "error ratio (source prints fractions; raw_value kept in percent)", "caveats": [ "Atypical (dysarthric) speech; ranks differ from typical-speech benchmarks.", "Instruct audio LLMs (Voxtral) were run without a text prompt in this condition.", "Per-utterance WER clipped at 1.0 per SAP protocol." ] }, { "id": "turkasr-bench-tr7-wer", "name": "TurkASR-Bench (7 Turkish sources, 5,200 utterances) · corpus WER", "category": "stt", "kind": "err", "url": "https://huggingface.co/spaces/yagmurtuncer/turkish-asr-leaderboard/blob/c27251669fb421135aa8dc2c138b802051775bc8/turkasr_bench.json", "metric": "word_error_rate", "setup": "Turkish ASR Leaderboard (yagmurtuncer), TurkASR-Bench version 2026-09-25: 743 utterances from each of 7 sources (Common Voice 17, FLEURS, Khan Academy, MediaSpeech, VoxForge, YouTube, Real-TurnTurk spontaneous dialogue; one source 742), fixed seed 42, 5,200 utterances in total. Corpus-level WER via jiwer over all 5,200 utterances ('ALL'), main normalization 'num': NFC + Turkish lowercase + punctuation removed (word-internal apostrophes kept) + digits verbalized in Turkish. All models run by the maintainer on the same set; per-source WERs kept as components.", "score_unit": "error ratio (source fraction)", "sample_count": 5200, "caveats": [ "Test set is gated on Hugging Face (datasets/yagmurtuncer/turkasr-bench); public components (Common Voice, FLEURS) may overlap training data.", "Maintainer also co-authors Turkish Whisper fine-tunes on the same board (not catalogue models).", "Corpus WER pooled over sources, so larger/longer sources weigh more than in a per-source mean." ] }, { "id": "voicecodebench-batch-wer", "name": "VoiceCodeBench · batch WER (300 English workplace segments)", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2608.28916v2", "metric": "word_error_rate", "sample_count": 300, "setup": "VoiceCodeBench v2 (Besimple AI): 300 human-recorded English workplace segments (5.59 h, 85 speakers) dense in structured tokens (emails, URLs, paths, commands, IDs, amounts), raw audio only (no prompts, keyterms or biasing). Batch/file APIs and local open models; corpus WER as printed in Table 5 (v2 scoring).", "score_unit": "error ratio (source % / 100)", "caveats": [ "v1 of the paper printed very different WERs for the same runs (different WER normalisation); v2 is pinned.", "Streaming configurations excluded (different products).", "Hosted API aliases may change after the run dates (May-Sep 2026).", "Whisper large-v3 was served by Groq (whisper-large-v3); Parakeet via transformers on Modal at a pinned HF revision." ] }, { "id": "voicecodebench-batch-ctem", "name": "VoiceCodeBench · batch canonical token/entity match (CTEM)", "category": "stt", "kind": "acc", "url": "https://arxiv.org/html/2608.28916v2", "metric": "entity_match_rate", "sample_count": 300, "setup": "Same batch runs as voicecodebench-batch-wer. CTEM = share of 1,482 audited target entities (26 types) whose exact canonical value is recoverable from the transcript, judged by a versioned GPT-5.5 recoverability verifier with an independent human audit. Higher is better.", "score_unit": "fraction of entities recovered (source % / 100)", "caveats": [ "LLM-verifier-based metric (audited).", "Measures exact value recovery, which ranks systems differently from WER (Spearman -0.28 per source)." ] }, { "id": "africodeswitch-en-yo-cap2000-wer", "name": "AfriCodeSwitch English-Yoruba code-switched speech · WER (cap2000)", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2609.11786v1", "metric": "word_error_rate", "sample_count": 2000, "setup": "2,000 shared utterances (cap2000) of English-Yoruba intra-sentential code-switched read speech from the AfriCodeSwitch validation corpus (100 speakers, 13 domains). Zero-shot greedy decoding; WER after NFC normalisation, case-folding and punctuation stripping (diacritic-sensitive). University of Florida authors' runs (Table II).", "score_unit": "error ratio (source % / 100)", "caveats": [ "None of the ASR systems supports Yoruba; Yoruba-token error is 97-99% for all, so WER (66-74%) mostly reflects English-token recognition. Low discriminative range.", "Whisper forced to English; the auto-language row is excluded." ] }, { "id": "rime-supportbench-pref-vs-coda", "name": "Rime SupportBench · paired human preference vs Rime Coda (US English)", "category": "tts", "kind": "acc", "url": "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/results/supportbench/supportbench_results.json", "metric": "paired_preference_vs_fixed_anchor", "setup": "Rime AlphaTTSBench v1.0.0 SupportBench: 750 US-English scripts (customer-support conversation turns (identifiers, amounts, dates, dialogue)); every competitor clip is paired with the same script rendered by Rime Coda (voice clementine). Blind US listeners on Podonos rate each pair on a five-point comparison scale with a real tie (+2 Rime much better .. -2 competitor much better); audio level-matched to -23 LUFS, 24 kHz. Ratings are averaged within script, scripts weighted equally; 95% family-clustered bootstrap intervals. One configured voice per provider (configs/provider_configs.json). Score = competitor-minus-Rime mean, so every competitor is measured against the same fixed anchor.", "scale_transform": "score = (d + 2) / 4, d = competitor-minus-Rime-Coda mean paired score (source reports Rime-minus-competitor; sign flipped)", "raw_scale_min": -2, "raw_scale_max": 2, "display_unit": "paired preference vs Rime Coda on -2..+2; 0.5 = parity with the anchor; not accuracy", "sample_count": 750, "setup_sources": [ "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/PROTOCOL.md", "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/configs/provider_configs.json", "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/results/README.md" ], "caveats": [ "Vendor-run: Rime (maker of the Coda anchor) ran all competitors; the anchor itself is not a catalogue model and gets no row.", "One voice per provider; results describe these voices/configurations, not every voice of a model.", "Hosted model aliases (sonic-3, gpt-4o-mini-tts, aura-2) can change server-side; collection finished before the 2026-09-12 freeze.", "The protocol forbids a combined score across benchmarks; SupportBench and ContentBench are separate series.", "ElevenLabs comparison is a separate 7-rating-per-script study (5,250 judgments) versus 2,730 for the others." ] }, { "id": "rime-contentbench-pref-vs-coda", "name": "Rime ContentBench · paired human preference vs Rime Coda (US English)", "category": "tts", "kind": "acc", "url": "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/results/contentbench/contentbench_results.json", "metric": "paired_preference_vs_fixed_anchor", "setup": "Rime AlphaTTSBench v1.0.0 ContentBench: 400 US-English scripts (narration of articles, stories and video scripts (incl. public-domain literary excerpts)); every competitor clip is paired with the same script rendered by Rime Coda (voice clementine). Blind US listeners on Podonos rate each pair on a five-point comparison scale with a real tie (+2 Rime much better .. -2 competitor much better); audio level-matched to -23 LUFS, 24 kHz. Ratings are averaged within script, scripts weighted equally; 95% family-clustered bootstrap intervals. One configured voice per provider (configs/provider_configs.json). Score = competitor-minus-Rime mean, so every competitor is measured against the same fixed anchor.", "scale_transform": "score = (d + 2) / 4, d = competitor-minus-Rime-Coda mean paired score (source reports Rime-minus-competitor; sign flipped)", "raw_scale_min": -2, "raw_scale_max": 2, "display_unit": "paired preference vs Rime Coda on -2..+2; 0.5 = parity with the anchor; not accuracy", "sample_count": 400, "setup_sources": [ "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/PROTOCOL.md", "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/configs/provider_configs.json", "https://github.com/rimelabs/alpha-tts-benchmark/blob/46c080779bf27b23ca2840dab7c2d7f19a945fa7/results/README.md" ], "caveats": [ "Vendor-run: Rime (maker of the Coda anchor) ran all competitors; the anchor itself is not a catalogue model and gets no row.", "One voice per provider; results describe these voices/configurations, not every voice of a model.", "Hosted model aliases (sonic-3, gpt-4o-mini-tts, aura-2) can change server-side; collection finished before the 2026-09-12 freeze.", "The protocol forbids a combined score across benchmarks; SupportBench and ContentBench are separate series." ] }, { "id": "voicefirst-indic10-overall-bt", "name": "Voice-First Nation · 10 Indian languages overall pairwise preference (Bradley–Terry)", "category": "tts", "kind": "elo", "url": "https://arxiv.org/html/2604.21481v2", "metric": "bradley_terry_rating", "setup": "5,357 native and code-mixed sentences in Bengali, Gujarati, Hindi, Kannada, Malayalam, Marathi, Odia, Tamil, Telugu and Urdu (16 domains; normalized, symbolic and code-mixed subsets; tongue twisters and STEM stress items). Seven systems generated with their recommended default voices, non-streaming, identical prompts, no style conditioning; same-gender voice pairs. 1,915 screened and trained native raters, >120K two-step pairwise comparisons (holistic preference locked before six perceptual axes). Overall preference fit with a maximum-likelihood Bradley–Terry model mapped to an Elo-like scale; 500 bootstrap resamples for 95% CIs.", "score_unit": "published Bradley–Terry rating on an Elo-like scale; higher is better", "setup_sources": [ "https://arxiv.org/html/2604.21481v2" ], "caveats": [ "Systems cover different language subsets (#lang 2-10); MiniMax Speech 2.8 HD was compared in only 2 languages (7,834 comparisons).", "'Gemini 2.5 Pro TTS' (rank 1) is not mapped: the label does not pin the Gemini API preview id vs the Cloud GA id.", "Bulbul v3 Beta (Sarvam) and Indic F5 are not catalogue models; their ratings anchor the scale but get no rows.", "Crowd pairwise preference from a controlled rater pool (not a public arena)." ] }, { "id": "instructttseval-en-avg", "name": "InstructTTSEval EN · mean instruction-following score (APS, DSD, RP)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2506.16381v1", "metric": "llm_judged_instruction_following_rate", "setup": "InstructTTSEval English subset: 1,000 expressive reference clips, each turned into three natural-language style instructions — Acoustic-Parameter Specification (APS), Descriptive-Style Directive (DSD) and Role-Play (RP). Systems synthesize from the text plus instruction only (no prompt audio). Gemini-as-a-judge scores whether each output follows the instruction (79% agreement with human annotators on a 50-clip check). Avg. = unweighted mean of the APS, DSD and RP percentages.", "score_unit": "fraction of instructions judged followed (source % / 100)", "setup_sources": [ "https://arxiv.org/html/2506.16381v1" ], "caveats": [ "LLM judge from the Gemini family also scores Gemini TTS outputs (possible self-preference).", "Unmarked asterisks in the source table on gemini-flash, gemini-pro and hume.", "Reference audio scores (84.3 EN / 82.5 ZH) and open instruct-TTS rows (Parler-TTS, VoxInstruct, PromptTTS, PromptStyle) not imported." ] }, { "id": "instructttseval-zh-avg", "name": "InstructTTSEval ZH · mean instruction-following score (APS, DSD, RP)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2506.16381v1", "metric": "llm_judged_instruction_following_rate", "setup": "InstructTTSEval Chinese subset: 1,000 expressive reference clips, each turned into three natural-language style instructions — Acoustic-Parameter Specification (APS), Descriptive-Style Directive (DSD) and Role-Play (RP). Systems synthesize from the text plus instruction only (no prompt audio). Gemini-as-a-judge scores whether each output follows the instruction (79% agreement with human annotators on a 50-clip check). Avg. = unweighted mean of the APS, DSD and RP percentages.", "score_unit": "fraction of instructions judged followed (source % / 100)", "setup_sources": [ "https://arxiv.org/html/2506.16381v1" ], "caveats": [ "LLM judge from the Gemini family also scores Gemini TTS outputs (possible self-preference).", "Unmarked asterisks in the source table on gemini-flash, gemini-pro and hume.", "Reference audio scores (84.3 EN / 82.5 ZH) and open instruct-TTS rows (Parler-TTS, VoxInstruct, PromptTTS, PromptStyle) not imported.", "Hume was not evaluated on the Chinese subset." ] }, { "id": "sap-voicerecon-bws-reconstruction", "name": "SAP voice reconstruction · human BWS reconstruction preference", "category": "tts", "kind": "elo", "url": "https://arxiv.org/html/2606.21343v1", "metric": "best_worst_scaling_plackett_luce", "setup": "Voice reconstruction for people with speech disorders: 193 English-L1 speakers from the Speech Accessibility Project (Dec-2024 release; Parkinson's, cerebral palsy, ALS, Down syndrome; 39 low-intelligibility). For each speaker one disordered recording is the zero-shot cloning prompt and another utterance's transcript is synthesized (193 utterances per system, 17 open-weight zero-shot systems). Subjective: Best–Worst Scaling with situational framing ('how the speaker sounded before the condition'), 4 systems per screen incl. the recording, 5 listening tests, 47 retained Prolific listeners (UK/US, English L1, paid); Plackett–Luce worths relative to the original recording.", "score_unit": "Elo-like rating = 1000 + 400·log10(e)·log-worth (log-worth relative to the original recording = 0)", "scale_transform": "score = 1000 + 173.7178 × published log-worth (natural-log Plackett–Luce worth; original recording = 1000)", "setup_sources": [ "https://arxiv.org/html/2606.21343v1" ], "caveats": [ "Special use case: cloning from disordered speech; not a general naturalness test.", "Qwen3-TTS, Fish Speech, VibeVoice and OpenVoice rows are not mapped (checkpoint/size/version not stated).", "Other rows (E2-TTS, F5-TTS, MaskGCT, VoiceCraft, GPT-SoVITS, HierSpeech, StyleTTS2, Vevo, WhisperSpeech) are not catalogue models.", "The Elo-like mapping is a fixed linear rescaling of the printed log-worths (no refit); raw log-worth kept per result." ] }, { "id": "sap-voicerecon-wer", "name": "SAP voice reconstruction · WER of reconstructed speech (Whisper turbo)", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2606.21343v1", "metric": "word_error_rate", "setup": "Voice reconstruction for people with speech disorders: 193 English-L1 speakers from the Speech Accessibility Project (Dec-2024 release; Parkinson's, cerebral palsy, ALS, Down syndrome; 39 low-intelligibility). For each speaker one disordered recording is the zero-shot cloning prompt and another utterance's transcript is synthesized (193 utterances per system, 17 open-weight zero-shot systems). Objective: WER of the synthetic speech transcribed with Whisper (turbo checkpoint).", "score_unit": "error ratio (source % / 100)", "setup_sources": [ "https://arxiv.org/html/2606.21343v1" ], "caveats": [ "Special use case: cloning from disordered speech; not a general naturalness test.", "Qwen3-TTS, Fish Speech, VibeVoice and OpenVoice rows are not mapped (checkpoint/size/version not stated).", "Other rows (E2-TTS, F5-TTS, MaskGCT, VoiceCraft, GPT-SoVITS, HierSpeech, StyleTTS2, Vevo, WhisperSpeech) are not catalogue models." ] }, { "id": "sap-voicerecon-ttsds-mean", "name": "SAP voice reconstruction · TTSDS dual-reference distributional score", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2606.21343v1", "metric": "ttsds_distributional_score", "setup": "Voice reconstruction for people with speech disorders: 193 English-L1 speakers from the Speech Accessibility Project (Dec-2024 release; Parkinson's, cerebral palsy, ALS, Down syndrome; 39 low-intelligibility). For each speaker one disordered recording is the zero-shot cloning prompt and another utterance's transcript is synthesized (193 utterances per system, 17 open-weight zero-shot systems). Objective: mean of TTSDS2 scores against two references — the SAP prompt utterances (speaker identity) and a 145-speaker LibriTTS subset (typical, intelligible speech).", "score_unit": "TTSDS score / 100", "display_unit": "distributional similarity 0-100; not accuracy", "setup_sources": [ "https://arxiv.org/html/2606.21343v1" ], "caveats": [ "Special use case: cloning from disordered speech; not a general naturalness test.", "Qwen3-TTS, Fish Speech, VibeVoice and OpenVoice rows are not mapped (checkpoint/size/version not stated).", "Other rows (E2-TTS, F5-TTS, MaskGCT, VoiceCraft, GPT-SoVITS, HierSpeech, StyleTTS2, Vevo, WhisperSpeech) are not catalogue models." ] }, { "id": "codecmos-accent-vctk-clone-nat-mos", "name": "CodecMOS-Accent VCTK voice clone · human naturalness MOS (S-NAT)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2603.14328v1", "metric": "mean_opinion_score_naturalness", "setup": "VCTK accented English: 32 speakers across 10 accents (20 F / 12 M), 5 target sentences per speaker (160 per system), zero-shot cloning from a different 3-7 s utterance of the same speaker, 16 kHz. 15 open-source voice-cloning systems. Crowdsourced listening test (Intergroup): 25 listeners (19 US, 2 CA, 3 EN, 1 SCO) each rated 784 samples on 5-point scales (19,600 annotations, ~4.9 per sample); 55 unusable samples removed. S-NAT: naturalness considering pronunciation, prosody and noise.", "scale_transform": "score = (raw MOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human MOS / 5; not accuracy", "sample_count": 160, "setup_sources": [ "https://arxiv.org/html/2603.14328v1" ], "caveats": [ "Ground-truth VCTK scores 4.045 S-NAT (below several TTS systems) because of recording artifacts.", "Orpheus-TTS not mapped (pretrained vs finetune checkpoint not stated); CosyVoice 2, Llasa-1B, VoiceStar, MaskGCT, VevoTTS, FireRedTTS, VALL-E-X were not mapped at import; NeuTTS Air was added on 2026-10-05.", "Checkpoint revisions not stated; open-weight defaults assumed." ] }, { "id": "codecmos-accent-vctk-clone-spk-mos", "name": "CodecMOS-Accent VCTK voice clone · human speaker-similarity MOS (S-SPK-SIM)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2603.14328v1", "metric": "mean_opinion_score_speaker_similarity", "setup": "VCTK accented English: 32 speakers across 10 accents (20 F / 12 M), 5 target sentences per speaker (160 per system), zero-shot cloning from a different 3-7 s utterance of the same speaker, 16 kHz. 15 open-source voice-cloning systems. Crowdsourced listening test (Intergroup): 25 listeners (19 US, 2 CA, 3 EN, 1 SCO) each rated 784 samples on 5-point scales (19,600 annotations, ~4.9 per sample); 55 unusable samples removed. S-SPK-SIM: whether test and reference sound like the same speaker.", "scale_transform": "score = (raw MOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human MOS / 5; not accuracy", "sample_count": 160, "setup_sources": [ "https://arxiv.org/html/2603.14328v1" ], "caveats": [ "Ground-truth VCTK scores 4.045 S-NAT (below several TTS systems) because of recording artifacts.", "Orpheus-TTS not mapped (pretrained vs finetune checkpoint not stated); CosyVoice 2, Llasa-1B, VoiceStar, MaskGCT, VevoTTS, FireRedTTS, VALL-E-X were not mapped at import; NeuTTS Air was added on 2026-10-05.", "Checkpoint revisions not stated; open-weight defaults assumed." ] }, { "id": "hpsv2-benchmark", "name": "HPS v2 benchmark (4-style average)", "category": "image", "kind": "acc", "chance": 0, "url": "https://github.com/tgxs002/HPSv2", "task": "text-to-image", "setup": { "url": "https://github.com/tgxs002/HPSv2/blob/866735ecaae999fa714bd9edfa05aa2672669ee3/README.md#the-hps-benchmark", "version": "HPS v2 benchmark ('v2 benchmark' table in the README, commit 866735e); HPD v2 benchmark prompts in Animation, Concept-art, Painting and Photo styles, scored with HPS v2 (not HPS v2.1)", "metric": "Published 'Averaged' HPS v2 score over the four styles. HPS v2 is a CLIP-based human-preference model; its score is reported x100 and stored divided by 100 (observed range about 0.23-0.28).", "protocol": "The benchmark authors generated every model's images themselves (settings in arXiv 2306.09341v2 Appendix D) and released them with HPD v2; SDXL 0.9 rows were added in July 2023.", "comparability": "Automatic preference-model score, not human votes. Do not pool with the README's HPS v2.1 table (same images, different scorer) or with HPS v2 values printed by other papers." }, "reported_scale": "HPS v2 x100", "benchmark_family": "hps" }, { "id": "tifa-v1", "name": "TIFA v1.0 (overall, mPLUG VQA)", "category": "image", "kind": "acc", "chance": 0, "url": "https://github.com/Yushi-Hu/tifa", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2303.11897v3#A2.T4", "version": "TIFA v1.0 (4,081 prompts, 25,829 pre-generated GPT-3 questions), report v3 Appendix Table 4", "metric": "Published Overall TIFA score: VQA accuracy of mPLUG-large on the benchmark questions for each generated image; 0-100 divided by 100.", "protocol": "Benchmark authors' own generations and scoring (Sec. 5.2, Appendix E).", "comparability": "Automatic VQA faithfulness score; not the TIFA160 human Likert ratings and not the image-only leaderboard in the repository. About two thirds of questions are yes/no, so the practical floor is well above 0." }, "reported_scale": "0-100", "benchmark_family": "tifa" }, { "id": "genai-arena-t2i", "name": "GenAI-Arena text-to-image (Elo, 4 Jun 2024)", "category": "image", "kind": "elo", "url": "https://arxiv.org/html/2406.04485v3#S3.T5", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2406.04485v3#S3.T5", "version": "GenAI-Arena report v3, Table 5(a); leaderboard last updated 4 June 2024; the full pool of 10 text-to-image models at that date", "metric": "Elo rating from anonymous pairwise human votes on the authors' arena; 95% CI kept in notes.", "protocol": "Prompts sampled from ImagenHub; models served through ImagenHub wrappers with the resolutions and step counts of Table 2; checkpoints per the arena's model registry.", "comparability": "Relative to this 10-model pool; not LMArena and not GEditBench Elo. The v4 (Oct 2024) top-10 table is a later snapshot of the same arena and is not imported." }, "reported_scale": "Elo", "benchmark_family": "genai-arena" }, { "id": "imagenhub-t2i-human", "name": "ImagenHub text-guided generation (human overall)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2310.01596v4#S5.T4", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2310.01596v4#S5.T4", "version": "ImagenHub report v4 (10 Mar 2024), Table 4, Text-guided Image Generation block", "metric": "Published Overall human score: average over images of sqrt(SC x PQ), where SC (semantic consistency) and PQ (perceptual quality) are rated 0 / 0.5 / 1; already 0-1.", "protocol": "Default settings from official implementations or Hugging Face docs, negative prompts and prompt engineering disabled; three raters per model (Prolific and the authors); ImagenHub standardized prompt set.", "comparability": "Human ratings on ImagenHub's own prompt set; do not pool with the editing or subject-driven blocks, or with automatic CLIP scores in the same table." }, "reported_scale": "0-1", "benchmark_family": "imagenhub" }, { "id": "evalmuse-fga-blip2-overall", "name": "EvalMuse-40K model ranking (FGA-BLIP2 overall alignment)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2412.18150v2#S9.T10", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2412.18150v2#S9.T10", "version": "EvalMuse-40K report v2 (25 Dec 2024), Appendix Table 10", "metric": "Published overall image-text alignment score predicted by the authors' FGA-BLIP2 model on the 1-5 Likert scale of the EvalMuse annotations; stored as (score - 1) / 4.", "protocol": "200 representative prompts; images generated by the authors for each model; single automatic scorer.", "comparability": "Automatic alignment metric trained on EvalMuse human labels, not a human rating of these images. Per-skill columns are correlated tracks and are not imported." }, "reported_scale": "1-5", "raw_scale_min": 1, "raw_scale_max": 5, "scale_transform": "score = (raw - 1) / 4", "display_unit": "FGA-BLIP2 predicted alignment / 5; not human MOS", "benchmark_family": "evalmuse" }, { "id": "kolorsprompts-human-overall", "name": "KolorsPrompts human overall satisfaction (Kolors team)", "category": "image", "kind": "acc", "chance": 0, "url": "https://github.com/Kwai-Kolors/Kolors/blob/038818d244ed103056abd10f429729a26af4d239/README.md#human-assessment", "task": "text-to-image", "setup": { "url": "https://github.com/Kwai-Kolors/Kolors/blob/038818d244ed103056abd10f429729a26af4d239/README.md#human-assessment", "version": "Kolors README Human Assessment table (April 2024 product versions); protocol in the Kolors technical report (imgs/Kolors_paper.pdf, same commit)", "metric": "Average Overall Satisfaction on a 1-5 scale (holistic rating with the prompt shown); stored as (score - 1) / 4.", "protocol": "KolorsPrompts (over 1,000 prompts, 14 categories); about 50 professional reviewers; four images per prompt, each rated five times; 1024x1024; Kolors prompted in Chinese, other models in English.", "limitations": "Evaluator is the maker of Kolors (Kolors row is lab). Visual Appeal and Text Faithfulness columns are correlated tracks of the same study and are not imported." }, "reported_scale": "1-5", "raw_scale_min": 1, "raw_scale_max": 5, "scale_transform": "score = (raw - 1) / 4", "display_unit": "mean human rating / 5", "benchmark_family": "kolorsprompts" }, { "id": "conceptmix-k2", "name": "ConceptMix (k=2 full-mark score)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2408.14339v1#S4.T4", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2408.14339v1#S4.T4", "version": "ConceptMix v1 (arXiv 2408.14339v1), Table 4, difficulty k=2", "metric": "Full-mark score: share of generated images that satisfy all k+1 = 3 sampled visual concepts, graded by GPT-4o; already 0-1.", "protocol": "300 prompts per k generated with GPT-4o from sampled concepts (objects, colors, numbers, shapes, sizes, textures, spatial relations, styles); open models from the Hugging Face checkpoints in Table 8; DALL·E 3 via API defaults.", "comparability": "Only k=2 is imported; k=1..7 are correlated difficulty levels of the same run (k=2 chosen as the middle of the non-floor range). Automatic GPT-4o grading." }, "reported_scale": "0-1", "benchmark_family": "conceptmix" }, { "id": "picovoice-en4-batch-avg-wer", "name": "Picovoice STT benchmark · English batch 4-set mean WER", "kind": "err", "category": "stt", "url": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md", "metric": "word_error_rate", "setup": "Picovoice/speech-to-text-benchmark README (Apache-2.0) at 43e7689: unweighted mean of LibriSpeech test-clean, LibriSpeech test-other, TED-LIUM and Common Voice (English) WER as printed in the 'Average' column; Picovoice's own normalizer; batch (non-streaming) engines on CPU. Different sets and normalizer from the Open ASR averages." }, { "id": "genai-arena-t2v", "name": "GenAI-Arena text-to-video (anonymous Elo)", "category": "video", "kind": "elo", "url": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv", "task": "text-to-video", "reported_scale": "Elo", "benchmark_family": "genai-arena-t2v", "setup": { "url": "https://huggingface.co/spaces/TIGER-Lab/GenAI-Arena/blob/4197b3a6b50aca8fae6c01d5968f08c3537024d1/arena_elo/results/20250322/video_generation_leaderboard.csv", "version": "Leaderboard snapshot arena_elo/results/20250322 (identical to results/latest) at Space revision 4197b3a", "metric": "'Arena Elo rating (anony)': Elo from anonymous public pairwise votes, computed by the maintainers' FastChat-derived arena_elo scripts; the Space displays this column (serve/leaderboard.py)", "protocol": "Public users submit a prompt; two anonymous models generate short clips (about 2-5 s) and the user votes. Models served through VideoGenHub (TIGER-AI-Lab, arena branch) or fal.ai endpoints listed in model/model_registry.py.", "limitations": "Pool of 15 mostly 2024 open models; ratings are relative to this pool and not on Arena's scale. Several models were retired from active battles before the snapshot. Allegro ran with 20 steps instead of its default 100." } }, { "id": "videophy-human-joint", "name": "VideoPhy (human, SA=1 & PC=1)", "category": "video", "kind": "acc", "chance": 0, "url": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md", "task": "text-to-video", "reported_scale": "0–100", "benchmark_family": "videophy", "setup": { "url": "https://github.com/Hritikbansal/videophy/blob/01d366c979e4a82875852adbf427ccc036e7373a/README.md", "version": "VideoPhy-1 human leaderboard, repository commit 01d366c", "metric": "Percentage of test prompts whose video is judged by human annotators to follow the caption (SA=1) and intuitive physics (PC=1); divided by 100.", "protocol": "VideoPhy-1 test captions (solid-solid, solid-fluid, fluid-fluid interactions); videos generated by the benchmark authors; human annotation (Appendix K of arXiv 2406.03520 lists per-model resolution/frames/steps).", "limitations": "Not the VideoPhy-1 auto-evaluator track (videophy-autoeval-avg) and not VideoPhy-2. Same videos as the auto track, so the two are correlated." } }, { "id": "love-aigve60k-perception-mos", "name": "AIGVE-60K human perception MOS (LOVE)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2505.12098v1#A6.T8", "task": "text-to-video", "reported_scale": "0–100 (rescaled z-score MOS)", "benchmark_family": "love-aigve60k", "setup": { "url": "https://arxiv.org/html/2505.12098v1#A6.T8", "version": "AIGVE-60K / LOVE, arXiv 2505.12098v1, Table 8 'Overall' column", "metric": "Model-level mean of per-video perception MOS; each rater's scores converted to z-scores and rescaled by 100(z+3)/6, then averaged over raters; divided by 100.", "protocol": "3,050 prompts across 20 task dimensions, 58,500 videos from 30 T2V models generated by the authors (commercial products accessed 2025-03-08), 15 annotators per video, ITU-R BT.500 outlier rejection.", "limitations": "Rescaled z-scores are relative to this pool, not an absolute quality scale. Perception and correspondence MOS rate the same videos and are correlated. Commercial rows without a version are not imported." } }, { "id": "love-aigve60k-correspondence-mos", "name": "AIGVE-60K human text-video correspondence MOS (LOVE)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2505.12098v1#A6.T9", "task": "text-to-video", "reported_scale": "0–100 (rescaled z-score MOS)", "benchmark_family": "love-aigve60k", "setup": { "url": "https://arxiv.org/html/2505.12098v1#A6.T9", "version": "AIGVE-60K / LOVE, arXiv 2505.12098v1, Table 9 'Overall' column", "metric": "Model-level mean of per-video text-video correspondence MOS (z-scores rescaled by 100(z+3)/6); divided by 100.", "protocol": "Same 58,500 videos and annotation campaign as the perception MOS.", "limitations": "Correlated with the perception track (same videos and raters). Commercial rows without a version are not imported." } }, { "id": "worldmodelbench-human-total", "name": "WorldModelBench (human total)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2502.20694v1#S3.T3", "task": "text-to-video / image-to-video", "reported_scale": "0–10", "raw_scale_min": 0, "raw_scale_max": 10, "benchmark_family": "worldmodelbench", "setup": { "url": "https://arxiv.org/html/2502.20694v1#S3.T3", "version": "WorldModelBench, arXiv 2502.20694v1, Table 3 (human annotations)", "metric": "Total score out of 10 (instruction following 0-3, common sense frame/temporal 0-2, five physics laws 0-5), averaged over prompts; divided by 10.", "protocol": "350 image+text condition pairs over 7 application domains; T2V models get the re-captioned text, I2V models the first frame plus text; authors' own generations with each model's recommended settings (Appendix 8.3); 67K human labels, up to 3 voters per video.", "limitations": "T2V and I2V models share one table; the conditioning is recorded per row. Judge-model scores (Table 9) and the Hard subset are not imported." } }, { "id": "magicarena-i2v-elo-seaweed", "name": "MagicArena image-to-video Elo (Seaweed-7B report)", "category": "video", "kind": "elo", "url": "https://arxiv.org/html/2504.08685v2#S4.T3", "task": "image-to-video", "reported_scale": "Elo", "benchmark_family": "magicarena-i2v", "setup": { "url": "https://arxiv.org/html/2504.08685v2#S4.T3", "version": "Seaweed-7B technical report, arXiv 2504.08685v2, Table 3", "metric": "Elo from side-by-side human votes on overall fidelity (win ratio retained in note).", "protocol": "MagicArena (ByteDance) pairwise evaluation: >500 raters, each model in >=7,000 comparisons; baseline videos generated with each model's official API as of early March 2025; Seaweed-7B at 480p/720p.", "limitations": "Evaluation run by the Seaweed developer (Seaweed-7B row is lab). Eight-model pool; ratings not on Arena's scale. Wan 2.1 and HunyuanVideo rows not imported (API vs checkpoint identity unclear)." } }, { "id": "videoverse-overall", "name": "VideoVerse overall (Gemini 2.5 Pro judge)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2510.08398v4#S3.T1", "task": "text-to-video", "reported_scale": "0–1608 (points)", "raw_scale_min": 0, "raw_scale_max": 1608, "benchmark_family": "videoverse", "setup": { "url": "https://arxiv.org/html/2510.08398v4#S3.T1", "version": "VideoVerse, arXiv 2510.08398v4 (15 May 2026), Table 1", "metric": "Published Overall points (793 binary hidden-semantics questions + event-following LCS over 815 events) divided by 1608.", "protocol": "300 prompts with 10 dimensions (event following, camera control, interaction, mechanics, material properties, natural constraints, common sense, attributes, 2D layout, 3D depth); Gemini 2.5 Pro evaluator validated by a user study (Table 3). Open models run by the authors on A800s; closed models via official APIs (Veo-3-fast, Sora-2 standard).", "limitations": "VLM-judged; 5 s short-video setting used where two settings exist. Sora-2 missing 4 videos." } }, { "id": "chronomagic-150-gpt4o-mtscore", "name": "ChronoMagic-Bench-150 GPT4o-MTScore (time-lapse metamorphosis)", "category": "video", "kind": "acc", "chance": 0.2, "url": "https://arxiv.org/html/2406.18522v2#S5.T4", "task": "text-to-video", "reported_scale": "1–5", "benchmark_family": "chronomagic-bench", "setup": { "url": "https://arxiv.org/html/2406.18522v2#S5.T4", "version": "ChronoMagic-Bench-150 subset, arXiv 2406.18522v2 Table 4", "metric": "GPT4o-MTScore: GPT-4o rates the metamorphic amplitude (how fully the requested time-lapse transformation happens) of uniformly sampled frames on a 5-point scale; model mean stored as value/5 with chance 0.2 mapping the scale floor to zero.", "protocol": "150 time-lapse prompts (biological, human-created, meteorological, physical); three videos per prompt; open models run by the authors with official code and default settings (Appendix D lists resolution/frames); closed products generated in June 2024.", "limitations": "Specialised to time-lapse/metamorphic prompts and scored by a GPT-4o judge (human agreement checked in Fig. 6). UMTScore, MTScore, CHScore and UMT-FVD from the same videos are not imported." } }, { "id": "arc-easy", "name": "ARC Easy", "kind": "acc", "chance": 0.25, "category": "llm", "url": "https://allenai.org/data/arc", "description": "AI2 Reasoning Challenge, Easy set (4-option science questions). Accuracy; not ARC Challenge. Shots and likelihood normalisation are source-specific and recorded per result." }, { "id": "sciq", "name": "SciQ", "kind": "acc", "chance": 0.25, "category": "llm", "url": "https://allenai.org/data/sciq", "description": "SciQ 4-option science exam questions (with support passage in the EleutherAI harness formulation). Accuracy; shots/normalisation recorded per result." }, { "id": "lambada-openai", "name": "LAMBADA (OpenAI version), last-word accuracy", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/EleutherAI/lm-evaluation-harness/blob/v0.0.1/lm_eval/tasks/lambada.py", "description": "Zero-shot last-word prediction accuracy on the 5,153-passage OpenAI-detokenised LAMBADA test file used by GPT-2/GPT-3 and the EleutherAI harness (task 'lambada' in harness v0, later 'lambada_openai'). Not the original LAMBADA 'standard' split and not perplexity." }, { "id": "triviaqa", "name": "TriviaQA", "kind": "acc", "chance": 0, "category": "llm", "url": "https://nlp.cs.washington.edu/triviaqa/", "description": "TriviaQA closed-book question answering, exact-match style accuracy. Split, shots and answer-matching rule differ by source and are recorded per result." }, { "id": "mathqa", "name": "MathQA", "kind": "acc", "chance": 0.2, "category": "llm", "url": "https://math-qa.github.io/", "description": "MathQA 5-option math word problems, multiple-choice accuracy by likelihood ranking (EleutherAI harness formulation). Zero-shot." }, { "id": "pubmedqa", "name": "PubMedQA", "kind": "acc", "chance": 0.3333, "category": "llm", "url": "https://pubmedqa.github.io/", "description": "PubMedQA labelled set (yes/no/maybe given an abstract), accuracy by likelihood ranking (EleutherAI harness formulation). Zero-shot." }, { "id": "copa", "name": "COPA", "kind": "acc", "chance": 0.5, "category": "llm", "url": "https://people.ict.usc.edu/~gordon/copa.html", "description": "Choice of Plausible Alternatives: pick the more plausible cause or effect of a premise (2 options). Accuracy; split, prompt template and answer-scoring rule recorded per result." }, { "id": "storycloze", "name": "Story Cloze Test", "kind": "acc", "chance": 0.5, "category": "llm", "url": "https://cs.rochester.edu/nlp/rocstories/", "description": "Story Cloze Test (Mostafazadeh et al.): choose the correct ending of a four-sentence story (2 options). Accuracy; split and scoring rule recorded per result." }, { "id": "commonsenseqa", "name": "CommonsenseQA", "kind": "acc", "chance": 0.2, "category": "llm", "url": "https://www.tau-nlp.sites.tau.ac.il/commonsenseqa", "description": "CommonsenseQA (Talmor et al. 2019) 5-option questions built from ConceptNet; accuracy (development set, since the test labels are not public). Shots and scoring rule recorded per result." }, { "id": "eq-bench-v1", "name": "EQ-Bench v1 (60 questions, paper run)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://arxiv.org/html/2312.06281v2", "metric": "eq_bench_v1_score_div_100", "description": "EQ-Bench v1: 60 GPT-4-written dialogues; the model rates the intensity of four emotions for a character; score is a distance to reference answers rescaled so 0 = random answering and 100 = reference (Paech 2023, Table 1), divided by 100. Critique-and-revision prompt, best of first-pass/revised score, temperature 0.01. Not EQ-Bench v2 (171 questions) or EQ-Bench 3. A scaled distance score, not an accuracy." }, { "id": "seed-tts-en-wavlm-sim", "name": "Seed-TTS test-en · speaker similarity (WavLM-large SV)", "category": "tts", "kind": "acc", "url": "https://github.com/BytedanceSpeech/seed-tts-eval", "metric": "speaker_embedding_cosine", "setup": "Zero-shot voice cloning on the standard Seed-TTS-Eval test-en set (1,088 Common Voice samples). Cosine similarity between WavLM-large speaker-verification embeddings (UniSpeech, as in seed-tts-eval) of the generated speech and the original target speech (SIM-o) or the prompt, as each paper states (recorded per result). Each paper runs its own generations; per-source calibration keeps runs separate.", "score_unit": "raw cosine similarity (0-1 in practice); percentages divided by 100", "display_unit": "speaker similarity cosine; not accuracy", "caveats": [ "Voice similarity, not quality or accuracy.", "Papers differ in inference settings, seeds and resampling; values from different papers are not pooled." ] }, { "id": "seed-tts-zh-wavlm-sim", "name": "Seed-TTS test-zh · speaker similarity (WavLM-large SV)", "category": "tts", "kind": "acc", "url": "https://github.com/BytedanceSpeech/seed-tts-eval", "metric": "speaker_embedding_cosine", "setup": "Zero-shot voice cloning on the standard Seed-TTS-Eval test-zh set (2,020 DiDiSpeech samples). Cosine similarity of WavLM-large speaker-verification embeddings (seed-tts-eval model) between generated speech and the original target (SIM-o) or prompt, as each paper states.", "score_unit": "raw cosine similarity; percentages divided by 100", "display_unit": "speaker similarity cosine; not accuracy", "caveats": [ "Voice similarity, not quality or accuracy.", "Papers differ in inference settings; values from different papers are not pooled." ] }, { "id": "seed-tts-en-utmos", "name": "Seed-TTS test-en · UTMOS (predicted MOS)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2510.12210v2", "metric": "predicted_mos_utmos", "setup": "Zero-shot voice cloning on Seed-TTS-Eval test-en; automatic naturalness prediction with UTMOS (Saeki et al., 2022).", "scale_transform": "score = (raw UTMOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "predicted MOS / 5; not human MOS, not accuracy", "caveats": [ "Automatic MOS predictor; ground-truth Seed-TTS test-en speech scores only 3.53 in the DiSTAR run." ] }, { "id": "seed-tts-en-human-smos", "name": "Seed-TTS test-en · human speaker-similarity MOS (SMOS)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2510.12210v2", "metric": "mean_opinion_score_speaker_similarity", "setup": "Listening test on Seed-TTS-Eval test-en generations: raters score similarity between synthesized speech and the prompt speaker on a 1-5 scale. Rater pool and sample count are paper-specific.", "scale_transform": "score = (raw SMOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human SMOS / 5; not accuracy", "caveats": [ "Small listening test; rater pool not comparable across papers.", "DiSTAR: human ground-truth SMOS 3.07; FireRedTTS CI ±0.58." ] }, { "id": "librispeech-pc-test-clean-f5subset-whisper-v3-wer", "name": "LibriSpeech-PC test-clean (F5-TTS 4-10 s subset) · Whisper large-v3 WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2410.06885v1", "metric": "word_error_rate", "setup": "Cross-sentence zero-shot TTS on the 4-10 s LibriSpeech-PC test-clean subset released with F5-TTS (1,127 utterances, 40 speakers/prompts, capitalisation and punctuation kept). WER from Whisper-large-v3.", "caveats": [ "Subset and prompt list released by the F5-TTS authors; papers re-run systems themselves." ] }, { "id": "librispeech-pc-test-clean-f5subset-wavlm-sim", "name": "LibriSpeech-PC test-clean (F5-TTS subset) · speaker similarity (WavLM SV)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2410.06885v1", "metric": "speaker_embedding_cosine", "setup": "Same generations as librispeech-pc-test-clean-f5subset-whisper-v3-wer. Cosine similarity from the WavLM-large (TDNN) speaker-verification model between generated speech and the original target speech (F5-TTS: SIM-o) or the reference prompt (DiSTAR), as recorded per result.", "score_unit": "raw cosine similarity", "display_unit": "speaker similarity cosine; not accuracy", "caveats": [ "Voice similarity, not quality or accuracy." ] }, { "id": "librispeech-pc-test-clean-f5subset-utmos", "name": "LibriSpeech-PC test-clean (F5-TTS subset) · UTMOS (predicted MOS)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2510.12210v2", "metric": "predicted_mos_utmos", "setup": "Same generations as the F5-subset WER benchmark; UTMOS automatic naturalness prediction.", "scale_transform": "score = (raw UTMOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "predicted MOS / 5; not human MOS, not accuracy", "caveats": [ "Automatic MOS predictor." ] }, { "id": "librispeech-test-clean-cross-hubert-wer", "name": "LibriSpeech test-clean 4-10 s cross-sentence · HuBERT-large WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2406.11427v2", "metric": "word_error_rate", "setup": "English zero-shot cross-sentence task (VALL-E protocol): LibriSpeech test-clean clips of 4-10 s, 3-s prompt from a different utterance of the same speaker. WER from CTC HuBERT-Large fine-tuned on LibriSpeech 960h (facebook/hubert-large-ls960-ft) with NeMo text normalisation.", "caveats": [ "Exact prompt list is paper-specific." ] }, { "id": "librispeech-test-clean-cross-wavlm-sim", "name": "LibriSpeech test-clean 4-10 s cross-sentence · SIM-o (WavLM-TDCNN)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2406.11427v2", "metric": "speaker_embedding_cosine", "setup": "Same generations as librispeech-test-clean-cross-hubert-wer; SIM-o from the WavLM-TDCNN speaker-verification model (UniSpeech) against the original target speech.", "score_unit": "raw cosine similarity", "display_unit": "speaker similarity cosine; not accuracy", "caveats": [ "Voice similarity, not quality or accuracy." ] }, { "id": "stylettszs-librispeech40-mos-n", "name": "StyleTTS-ZS LibriSpeech-40 listening test · naturalness MOS", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2409.10058v1", "metric": "mean_opinion_score_naturalness", "setup": "40 LibriSpeech test-clean sentences with 3-s reference prompts (the VALL-E / NaturalSpeech 2 / FlashSpeech prompt set); each system rated by 10 US native-English MTurk raters on 1-5 in 0.5 steps, systems presented side by side with hidden labels (MUSHRA-like). Public LibriTTS-trained checkpoints.", "scale_transform": "score = (raw MOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human MOS / 5; not accuracy", "sample_count": 40, "caveats": [ "Run by the StyleTTS 2 / StyleTTS-ZS authors (their StyleTTS-ZS rows not imported).", "Small sample (40 sentences, 10 raters).", "Share-alike source (CC BY-SA 4.0)." ] }, { "id": "stylettszs-librispeech40-mos-s", "name": "StyleTTS-ZS LibriSpeech-40 listening test · speaker-similarity MOS", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2409.10058v1", "metric": "mean_opinion_score_speaker_similarity", "setup": "Same listening test as stylettszs-librispeech40-mos-n; similarity to the prompt speaker, 1-5 in 0.5 steps.", "scale_transform": "score = (raw MOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human MOS / 5; not accuracy", "sample_count": 40, "caveats": [ "Small sample (40 sentences, 10 raters).", "Share-alike source (CC BY-SA 4.0)." ] }, { "id": "stylettszs-librispeech40-hubert-wer", "name": "StyleTTS-ZS LibriSpeech-40 · HuBERT-large WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2409.10058v1", "metric": "word_error_rate", "setup": "WER of the same 40 generated LibriSpeech test-clean sentences, transcribed with facebook/hubert-large-ls960-ft.", "caveats": [ "Only 40 sentences.", "Share-alike source (CC BY-SA 4.0)." ] }, { "id": "voicecraft-zs250-whisper-medium-wer", "name": "VoiceCraft zero-shot TTS set (LibriTTS + YouTube, 250) · Whisper medium.en WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2403.16973v3", "metric": "word_error_rate", "setup": "250 prompt-transcript pairs, half LibriTTS and half GigaSpeech YouTube test; ~3-s prompts cut at word boundaries, targets 8-40 words from another utterance of the same speaker. WER by Whisper medium.en.", "caveats": [ "VoiceCraft generates 5 samples and keeps the shortest (its own row only).", "Share-alike source (CC BY-SA 4.0)." ] }, { "id": "voicecraft-zs250-wavlm-sim", "name": "VoiceCraft zero-shot TTS set (250) · speaker similarity (WavLM-TDCNN)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2403.16973v3", "metric": "speaker_embedding_cosine", "setup": "Same 250 generations; SIM from WavLM-TDCNN speaker embeddings.", "score_unit": "raw cosine similarity", "display_unit": "speaker similarity cosine; not accuracy", "caveats": [ "Share-alike source (CC BY-SA 4.0)." ] }, { "id": "voicecraft-zs80-human-nmos", "name": "VoiceCraft zero-shot TTS listening test (80) · naturalness MOS", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2403.16973v3", "metric": "mean_opinion_score_naturalness", "setup": "80 utterances (40 LibriTTS, 40 YouTube) sampled from the 250-pair set; 10 MTurk ratings (US residents) per utterance on a 5-point Likert scale; Total column.", "scale_transform": "score = (raw MOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human MOS / 5; not accuracy", "sample_count": 80, "caveats": [ "Ground truth scores 4.48; YouTube ground truth rated above LibriTTS.", "Share-alike source (CC BY-SA 4.0)." ] }, { "id": "voxcpm-listening-zh-nmos", "name": "VoxCPM listening test · Chinese naturalness MOS (N-MOS)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2509.24650v1", "metric": "mean_opinion_score_naturalness", "setup": "VoxCPM report subjective evaluation: 20 native speakers rate naturalness on a 5-point scale; zero-shot cloning outputs of each system. Test sentences are not itemised.", "scale_transform": "score = (raw MOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human MOS / 5; not accuracy", "caveats": [ "Run by the VoxCPM authors (VoxCPM rows are lab).", "Test set composition not stated." ] }, { "id": "voxcpm-listening-zh-smos", "name": "VoxCPM listening test · Chinese speaker-similarity MOS (S-MOS)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2509.24650v1", "metric": "mean_opinion_score_speaker_similarity", "setup": "Same listening test; speaker similarity to the prompt on a 5-point scale (Chinese).", "scale_transform": "score = (raw MOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human MOS / 5; not accuracy", "caveats": [ "Run by the VoxCPM authors." ] }, { "id": "voxcpm-listening-en-nmos", "name": "VoxCPM listening test · English naturalness MOS (N-MOS)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2509.24650v1", "metric": "mean_opinion_score_naturalness", "setup": "Same listening test, English: naturalness on a 5-point scale by native speakers.", "scale_transform": "score = (raw MOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human MOS / 5; not accuracy", "caveats": [ "Run by the VoxCPM authors." ] }, { "id": "voxcpm-listening-en-smos", "name": "VoxCPM listening test · English speaker-similarity MOS (S-MOS)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2509.24650v1", "metric": "mean_opinion_score_speaker_similarity", "setup": "Same listening test, English: speaker similarity on a 5-point scale.", "scale_transform": "score = (raw MOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human MOS / 5; not accuracy", "caveats": [ "Run by the VoxCPM authors." ] }, { "id": "cv3-eval-en-wer", "name": "CV3-Eval EN (zero-shot English) · WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2605.20830v2", "metric": "word_error_rate", "setup": "CV3-Eval (CosyVoice 3 release) zero-shot voice-cloning English subset (500 items), official protocol: Whisper-large-v3 WER.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Evaluator re-runs released checkpoints with default settings; F5-TTS failed on 15/500 long texts in the Raon run and is not imported." ] }, { "id": "cv3-eval-hard-en-eres2net-sim", "name": "CV3-Eval hard-en · speaker similarity (ERes2Net SV)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2605.20830v2", "metric": "speaker_embedding_cosine", "setup": "CV3-Eval hard-en subset (linguistically complex English targets), zero-shot cloning; speaker similarity from an ERes2Net speaker-verification model per the official CV3-Eval protocol.", "score_unit": "raw cosine similarity", "display_unit": "speaker similarity cosine; not accuracy", "caveats": [ "Voice similarity, not quality or accuracy." ] }, { "id": "cv3-eval-hard-en-dnsmos", "name": "CV3-Eval hard-en · DNSMOS (predicted quality)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2605.20830v2", "metric": "predicted_mos_dnsmos", "setup": "CV3-Eval hard-en generations scored with DNSMOS (Microsoft non-intrusive predicted MOS) per the official CV3-Eval protocol.", "scale_transform": "score = (raw DNSMOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "predicted MOS / 5; not human MOS, not accuracy", "caveats": [ "Automatic quality predictor designed for noise suppression; not a naturalness judgement." ] }, { "id": "raon-opentts-eval-overall-wer", "name": "Raon-OpenTTS-Eval · overall WER (4 acoustic conditions)", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2605.20830v2", "metric": "word_error_rate", "setup": "Raon-OpenTTS-Eval: 6,000 English prompt-text pairs (500 per corpus) from 12 corpora in four conditions: Clean (LibriSpeech-clean, ST American English, CMU-ARCTIC, L2-ARCTIC, VCTK), Noisy (LibriSpeech-other, TED-LIUM 3), Wild (AMI-IHM, AMI-SDM zero-WER segments) and Expressive (Expresso, CREMA-D, EmoV-DB); target text from a disjoint sentence of the same dataset; WER after the Whisper text normaliser, pooled over all samples.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Wild-condition WER can exceed 100% for some systems and dominates the pooled mean.", "Benchmark built by the Raon authors, whose own models are lab rows." ] }, { "id": "raon-opentts-eval-overall-wavlm-sim", "name": "Raon-OpenTTS-Eval · overall speaker similarity (WavLM)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2605.20830v2", "metric": "speaker_embedding_cosine", "setup": "Same generations as raon-opentts-eval-overall-wer; cosine similarity between WavLM speaker embeddings of the prompt and generated audio, pooled over all samples.", "score_unit": "raw cosine similarity", "display_unit": "speaker similarity cosine; not accuracy", "caveats": [ "Voice similarity, not quality or accuracy." ] }, { "id": "raon-opentts-eval-human-smos", "name": "Raon-OpenTTS-Eval · human speaker-similarity MOS (overall)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2605.20830v2", "metric": "mean_opinion_score_speaker_similarity", "setup": "30 randomly sampled items per acoustic condition (Clean/Noisy/Wild/Expressive), each rated by 6 US MTurk annotators on a 5-point scale for similarity to the reference (speaker, style, environment); Overall column.", "scale_transform": "score = (raw SMOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human SMOS / 5; not accuracy", "sample_count": 120, "caveats": [ "Small listening test; raters also judge environment and style similarity.", "CMOS results (relative to Raon-OpenTTS-1B) not imported." ] }, { "id": "rvcbench-libritts-ecapa-sim", "name": "RVCBench LibriTTS clean voice cloning · ECAPA speaker similarity", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2602.00443v2", "metric": "speaker_embedding_cosine", "setup": "RVCBench (UBC / Vector Institute) clean English-VC baseline: speaker-balanced LibriTTS subset (20 F + 20 M test-clean speakers, supplemented from dev-clean; 100 utterances each, 4k waveforms, 24 kHz) with reference audio and target text re-paired into new zero-shot cloning instances. SIM = SpeechBrain ECAPA-TDNN (speechbrain/spkrec-ecapa-voxceleb) verification score between ground-truth and generated audio, mean over utterances with 1000-sample bootstrap. All open-weight systems run by the authors with default inference settings via the RVCBench toolkit.", "score_unit": "raw cosine similarity (not accuracy)", "setup_sources": [ "https://arxiv.org/html/2602.00443v2", "https://github.com/Nanboy-Ronan/RVCBench" ], "caveats": [ "Robustness benchmark: the clean LibriTTS condition is its baseline, not a curated clean-TTS test set; prompts are re-paired and configs use a default prompt transcript when none is supplied.", "Model checkpoints identified from the CC0 RVCBench repository configs (initial upload for v1 models; commit 171c7cb for models added in v2), not from the paper table itself.", "FishSpeech (config loads openaudio-s1-mini while the paper describes FishSpeech 390M), StyleTTS (paper says original StyleTTS, config loads StyleTTS2), IndexTTS (version not pinned), GLM-TTS (local checkpoint dir), PlayDiffusion and OZSpeech rows not imported.", "MOS column not imported: the paper cites DNSMOS in the main text but defines both UTMOS and DNSMOS in the appendix." ] }, { "id": "wayu-thai-cer500-typhoon-whisper", "name": "Wayu Thai TTS eval · 500-utterance Thai CER (Typhoon Whisper Large V3)", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2609.03502v1", "metric": "character_error_rate", "setup": "500 Thai transcripts (250 WangchanThaiInstruct test split: retail/finance/medical/legal; 250 Common Voice 17.0 Thai validated test) synthesised by each system; transcribed with typhoon-ai/typhoon-whisper-large-v3; CER after normalization and whitespace removal, per-utterance CER capped at 100%, mean reported in percent (divided by 100 here).", "score_unit": "fraction (source percent / 100)", "setup_sources": [ "https://arxiv.org/html/2609.03502v1", "https://github.com/wayu-research/thai-tts-eval" ], "caveats": [ "Run by Wayu Research / Paxa Labs / Typhoon, who make Wayu-Paxa-TTS-Edge (lab) and used OmniVoice as its teacher; Gemini 3.1 Flash TTS uses a prebuilt voice while the others use one fixed/cloned voice.", "Transcriber is a Typhoon (same organisation) Thai Whisper fine-tune." ] }, { "id": "wayu-thai-challenge-keyword-acc", "name": "Wayu Thai TTS eval · 1,531-item Challenge-Set keyword accuracy", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2609.03502v1", "metric": "keyword_accuracy", "setup": "1,531 held-out Thai sentences, each containing one hard target expression (curated Thai-English terms, WangchanThaiInstruct, VISTEC-TP-TH-2021 annotations, PyThaiNLP place names, Thai names, long sentences). Item correct when the normalized typhoon-whisper-large-v3 transcript contains the expected form or an authorised alternate (exact substring match, with or without spaces).", "score_unit": "fraction of items correct", "setup_sources": [ "https://arxiv.org/html/2609.03502v1", "https://github.com/wayu-research/thai-tts-eval" ], "caveats": [ "Same authors/transcriber caveats as wayu-thai-cer500-typhoon-whisper.", "ASR-based exact-match proxy for pronunciation correctness, not human judgement." ] }, { "id": "cloneval-ls-test-clean-wavlm-sim", "name": "ClonEval · LibriSpeech test-clean voice-clone speaker similarity (WavLM)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/spaces/amu-cai/Open_Voice_Cloning_Leaderboard/blob/f715ec805e912f43e79a5b948191ff044df27fec/results.csv", "metric": "speaker_embedding_cosine", "setup": "ClonEval benchmark (AMU): zero-shot voice cloning of LibriSpeech test-clean speakers; each reference paired with a different prompt text from the same dataset; mean cosine similarity of microsoft/wavlm-base-plus-sv embeddings between reference and generated sample. Evaluated by the ClonEval authors; Open Voice Cloning Leaderboard results.csv.", "score_unit": "raw cosine similarity (0-1); not accuracy", "setup_sources": [ "https://huggingface.co/spaces/amu-cai/Open_Voice_Cloning_Leaderboard/blob/f715ec805e912f43e79a5b948191ff044df27fec/results.csv", "https://arxiv.org/abs/2504.20581v3" ], "caveats": [ "Speaker similarity only; says nothing about intelligibility or naturalness.", "Only five systems; SpeechT5 (label/script conflict) and WhisperSpeech (checkpoint unstated) not mapped." ] }, { "id": "arabic-tts-arena-elo-20260924", "name": "Arabic TTS Arena · community blind A/B rating, 2026-09-24 snapshot", "category": "tts", "kind": "elo", "url": "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json", "metric": "arena_rating", "setup": "Navid-AI Arabic TTS Arena (Hugging Face Space): blind pairwise votes on Arabic prompts across MSA, Gulf, Egyptian, Levantine and Maghrebi dialects; 8,647 votes at the pinned leaderboard.json (last_updated 2026-09-24). Published overall elo with CI; only active (non-retired) models from the current fit are imported.", "score_unit": "published arena rating (Elo scale)", "setup_sources": [ "https://huggingface.co/spaces/Navid-AI/Arabic-TTS-Arena/blob/6e003a592d4b5e64ce315fb635da22ec57f2fa82/leaderboard.json" ], "caveats": [ "Arabic only; commercial models served through third-party routes as listed by the arena.", "Rating method not documented beyond \"ELO from community votes\" with CI; retired models keep a rating that is still refitted from all votes (Fish Speech S1-mini moved 965.5 → 931.4 across snapshots on the same 506 battles).", "Voxtral TTS (3 battles) and Arabic-specific non-catalogue systems (Coda-V2, AIC TTS, Audar, SILMA, Fasih, Nabra) anchor the scale but get no rows." ] }, { "id": "balacoon-vctk-zero-tts-whisper-cer", "name": "balacoon zero-shot TTS · VCTK · Whisper-large-v3-turbo CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/README.md", "metric": "character_error_rate", "setup": "balacoon speech_gen_eval zero-tts protocol on VCTK 2k subset (studio read speech), n=2000: reference-audio voice cloning; transcripts by openai/whisper-large-v3-turbo; CER as a fraction.", "score_unit": "error ratio (source fraction)", "sample_count": 2000, "setup_sources": [ "https://huggingface.co/datasets/balacoon/speech_gen_eval_testsets/blob/cbafef11262fba69c16d3ff9c907b59daad1adae/README.md", "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/src/speech_gen_eval/whisperv3_intelligibility.py" ], "caveats": [ "Independent evaluator (balacoon); systems evaluated on different dates (2025-02 to 2026-01) with the same speech_gen_eval metric set.", "Some systems may have VCTK in training data (test-set README)." ] }, { "id": "balacoon-vctk-zero-tts-redimnet-sim", "name": "balacoon zero-shot TTS · VCTK · ReDimNet speaker similarity", "category": "tts", "kind": "acc", "url": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/README.md", "metric": "speaker_embedding_cosine", "setup": "Same generations; cosine similarity of ReDimNet speaker embeddings between reference audio and generated speech (redimnet_secs).", "score_unit": "raw cosine similarity; not accuracy", "sample_count": 2000, "caveats": [ "Same audio as the CER and UTMOS rows." ] }, { "id": "balacoon-daps-celeb-zero-tts-whisper-cer", "name": "balacoon zero-shot TTS · DAPS-celeb · Whisper-large-v3-turbo CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/README.md", "metric": "character_error_rate", "setup": "balacoon speech_gen_eval zero-tts protocol on DAPS-celeb (1,881 DAPS consumer-mic source utterances, celebrity voice references), n=1881: reference-audio voice cloning; transcripts by openai/whisper-large-v3-turbo; CER as a fraction.", "score_unit": "error ratio (source fraction)", "sample_count": 1881, "setup_sources": [ "https://huggingface.co/datasets/balacoon/speech_gen_eval_testsets/blob/cbafef11262fba69c16d3ff9c907b59daad1adae/README.md", "https://github.com/balacoon/speech_gen_eval/blob/3ec51a00e16ac5708b8c2a47690f6e988baa7d8e/src/speech_gen_eval/whisperv3_intelligibility.py" ], "caveats": [ "Hard condition: consumer-microphone source text with internet celebrity references; XTTS-v2 inputs were split at 250 characters." ] }, { "id": "balacoon-daps-celeb-zero-tts-redimnet-sim", "name": "balacoon zero-shot TTS · DAPS-celeb · ReDimNet speaker similarity", "category": "tts", "kind": "acc", "url": "https://huggingface.co/datasets/balacoon/speech_gen_baselines/blob/e14eac6aa6752aa68cb7b7ebfc379d51a1a085fd/README.md", "metric": "speaker_embedding_cosine", "setup": "Same generations; cosine similarity of ReDimNet speaker embeddings between reference audio and generated speech (redimnet_secs).", "score_unit": "raw cosine similarity; not accuracy", "sample_count": 1881, "caveats": [ "Same audio as the CER and UTMOS rows." ] }, { "id": "librispeech-test-clean-zero-tts-hubert-wer", "name": "LibriSpeech test-clean zero-shot TTS · HuBERT-large WER", "category": "tts", "kind": "err", "url": "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md", "metric": "word_error_rate", "setup": "Zero-shot voice cloning on LibriSpeech test-clean as evaluated by the MaskGCT authors (Amphion): transcripts by facebook/hubert-large-ls960-ft, WER; speaker prompts from the same speaker. Exact utterance subset not stated in the README.", "score_unit": "error ratio (source % / 100)", "setup_sources": [ "https://github.com/open-mmlab/Amphion/blob/26f6883110181f1dbfe95c70a7c7dbaf4de5f42a/models/tts/maskgct/README.md", "https://arxiv.org/abs/2409.00750v3" ], "caveats": [ "Subset/duration filter not stated; not comparable across papers that use LibriSpeech-PC or 4-10 s subsets.", "Only MaskGCT (lab) plus re-run XTTS-v2 and VoiceCraft rows are imported." ] }, { "id": "ttsds2-listening-mos", "name": "TTSDS2 listening test · naturalness MOS (4 English domains)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv", "metric": "mean_opinion_score_naturalness", "setup": "TTSDS2 study (Minixhofer et al.): 20 open-weight voice-cloning systems (2022-2024 releases, latest checkpoint before 2025-01-01) synthesise 60 speakers per domain in four English domains: Clean (LibriTTS test), Noisy (LibriTTS with added noise), Wild (YouTube speech) and Kids (children's speech). The first utterance of each speaker is the reference, the transcript of a second utterance is synthesised. Prolific listeners (UK/US native speakers, headphones, three attention checks passed; ~50 per domain, one domain each) rate naturalness on a 5-point Bad-Excellent scale. Score = unweighted mean over the four domains of the per-domain mean rating, computed by us from the individual ratings (rating_type=mos). This reproduces the paper's Table 2 MOS column to two decimals for every system.", "scale_transform": "score = (raw MOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human MOS / 5; not accuracy", "setup_sources": [ "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/README.md", "https://arxiv.org/abs/2506.19441v2", "https://github.com/ttsds/systems/tree/c5836668e3e0f1c1655732d462a06a1814098851/models" ], "caveats": [ "Ground-truth ratings are not published (licensing); the paper reports ground truth at MOS 3.70, so absolute MOS levels are survey-specific.", "Each MOS page mixes one ground-truth clip with four random systems; roughly 200-280 ratings per system in total.", "ParlerTTS was run through a codebase modification to allow voice cloning (paper Appendix A).", "XTTS (v1/v2 identity conflict) and VoiceCraft-830M (two 830M checkpoints) are not mapped.", "CMOS from the same survey is not imported (same naturalness construct and listeners)." ] }, { "id": "ttsds2-listening-smos", "name": "TTSDS2 listening test · speaker-similarity MOS (4 English domains)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/subjective_results.csv", "metric": "mean_opinion_score_speaker_similarity", "setup": "Same TTSDS2 survey, systems, speakers and domains as ttsds2-listening-mos. SMOS: listeners compare a synthetic sample with the ground-truth recording of the same speaker (18 comparisons per listener) and rate speaker similarity on a 5-point scale. Score = unweighted mean over Clean/Noisy/Wild/Kids of the per-domain mean rating (rating_type=smos), computed by us from the individual ratings; reproduces the paper's Table 2 SMOS column to two decimals.", "scale_transform": "score = (raw SMOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "human similarity MOS / 5; not accuracy", "setup_sources": [ "https://huggingface.co/datasets/ttsds/listening_test/blob/4d4d98f452b17155d8bf9b44a62754407e21dd96/README.md", "https://arxiv.org/abs/2506.19441v2" ], "caveats": [ "Voice-cloning fidelity, not naturalness; same listeners and samples as the MOS benchmark.", "Ground-truth SMOS 4.37 in the paper (not published in the dataset).", "XTTS and VoiceCraft-830M not mapped (see MOS benchmark)." ] }, { "id": "ttsds2-multilingual-en-overall", "name": "TTSDS2 multilingual benchmark · English overall (YouTube, 2025 round)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/en_aggregated_multilingual.csv", "metric": "distribution_similarity_score", "setup": "TTSDS2 multilingual round published May 2025 (ttsds/leaderboard_data). The TTSDS2 pipeline collects YouTube talk-show/interview/news/podcast speech uploaded after the newest evaluated system was published, diarises it with Whisper, filters controversial content, crosstalk and music, and keeps 50 speaker-matched utterance pairs per language (reference set / synthesis set). Every system clones each reference speaker and speaks the paired transcript. Multilingual TTSDS2 (mHuBERT-147 / XLS-R features) scores each factor 0-100 by distance to the held-out real speech ('Ground Truth A') versus noise distributions; OVERALL is the unweighted mean of SPEAKER, PROSODY, INTELLIGIBILITY and GENERIC. A second real-speech set ('Ground Truth B en') scores 91.63. Divided by 100.", "score_unit": "source 0-100 divided by 100", "setup_sources": [ "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/README.md", "https://arxiv.org/abs/2506.19441v2", "https://github.com/ttsds/daisy/tree/941e9cc4ea66ff1f0b15de63aa353ab7abd576f1" ], "caveats": [ "Distribution similarity to real speech, not recognition accuracy or MOS; distinct from TTSDS v1 and from ZTTS1's TTSDS2 prosody factor.", "Same 20 systems as the TTSDS2 listening test; XTTS and VoiceCraft-830M not mapped.", "Scores are clustered (79-90); small differences are within run-to-run noise." ] }, { "id": "ttsds2-multilingual-zh-overall", "name": "TTSDS2 multilingual benchmark · Chinese overall (YouTube, 2025 round)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/ttsds/zh_aggregated_multilingual.csv", "metric": "distribution_similarity_score", "setup": "Same TTSDS2 multilingual round and scoring as ttsds2-multilingual-en-overall, Chinese (zh) reference speakers and texts; the six systems in the round that support Chinese. OVERALL = unweighted mean of SPEAKER, PROSODY, INTELLIGIBILITY and GENERIC; Ground Truth B scores 91.51. Divided by 100.", "score_unit": "source 0-100 divided by 100", "setup_sources": [ "https://huggingface.co/datasets/ttsds/leaderboard_data/blob/09ee7f1fb4cb32bb1ae187dfb4f6b7606e0ede69/README.md", "https://arxiv.org/abs/2506.19441v2" ], "caveats": [ "Distribution similarity, not accuracy or MOS.", "Only six systems; OpenVoice V2 runs through MeloTTS Chinese base speakers." ] }, { "id": "geckonum-task1-exact", "name": "GeckoNum Task 1: exact number generation (human-annotated)", "category": "image", "kind": "acc", "chance": 0.1, "url": "https://arxiv.org/html/2406.14774v3#S5.T2", "task": "text-to-image", "reported_scale": "0-100 (%)", "benchmark_family": "geckonum", "setup": { "url": "https://arxiv.org/html/2406.14774v3#S4", "version": "GeckoNum, arXiv 2406.14774v3, Table 2 Task 1 column", "metric": "Share of image-question pairs where the annotated object count equals the number in the prompt (per-response binary accuracy averaged); percentage divided by 100. Random chance 10.0 per the table.", "protocol": "Numeric-simple, attribute and compositional count prompts (Task 1 of 3); five images per model and prompt (five seeds); each image annotated by five paid participants; counts pre-processed as in Appendix C.2.", "comparability": "Only Task 1 is imported; Tasks 2 (approximate/zero) and 3 (conceptual) are separate, correlated tasks of the same study. Standard errors (~0.4-0.5 pp) kept in notes." } }, { "id": "worldreasonbench-human-elo", "name": "WorldReasonBench human Elo (image+text-to-video world-state prediction)", "category": "video", "kind": "elo", "url": "https://arxiv.org/html/2605.10434v1#S4.T4", "task": "image-to-video", "reported_scale": "Elo", "benchmark_family": "worldreasonbench", "setup": { "url": "https://arxiv.org/html/2605.10434v1#S4.SS3", "version": "WorldReasonBench / WorldRewardBench, arXiv 2605.10434v1, Table 4 'Human Elo'", "metric": "Elo-scale ability from a Bradley-Terry model with Davidson ties fitted to expert pairwise preferences (Appendix G; 1,000 prompt-cluster bootstrap resamples for CIs).", "protocol": "436 TI2V cases (initial image + instruction; 4 reasoning dimensions, 22 sub-categories); WorldRewardBench subset: 11 generators, 8 videos sampled per case, 15 trained annotators rate reasoning quality, temporal consistency and aesthetics on 1-5 (S = 0.4 s_r + 0.3 s_c + 0.3 s_a), ranked within case into ~6K preference pairs over ~1.4K videos (near-equal pairs as ties). Closed models via their commercial APIs; open models with each official inference recipe, 480p-720p clips.", "limitations": "Ratings are relative to this 11-generator pool, not LMArena's scale. Pairs are derived from absolute 1-5 ratings rather than direct side-by-side votes. Kling and UniVideo rows are not imported; Sora 2 at 12 s is a second setting of the same model." } }, { "id": "cruxeval-i", "name": "CRUXEval-I", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/facebookresearch/cruxeval/blob/190faf16d175b5847b0af05d937872b1fb395942/README.md", "description": "CRUXEval input prediction: given a short Python function and an output, produce an input that yields it (800 problems); pass@1, direct prediction without chain-of-thought unless the note says otherwise. Companion of cruxeval-o." }, { "id": "ds-1000", "name": "DS-1000", "kind": "acc", "chance": 0, "category": "llm", "url": "https://arxiv.org/html/2402.19173v1", "description": "DS-1000 data-science code generation (1,000 problems over NumPy, Pandas, Matplotlib, PyTorch, SciPy, Scikit-Learn, TensorFlow); overall pass@1, completion format unless the note says otherwise." }, { "id": "evoeval", "name": "EvoEval (overall)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://arxiv.org/html/2403.19114v1", "description": "EvoEval overall pass@1 as printed in the EvoEval paper (Table 2): mean of HumanEval and the five semantic-altering evolved benchmarks Difficult, Creative, Subtle, Combine and Tool_Use (100 problems each); greedy decoding." }, { "id": "humanevalsynthesize", "name": "HumanEvalSynthesize (HumanEvalPack, 6-language average)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://arxiv.org/html/2405.04324v1", "description": "HumanEvalPack synthesis task (Muennighoff et al. 2023): pass@1 averaged over Python, JavaScript, Java, Go, C++ and Rust as printed by the source; greedy. Not the Python-only original HumanEval." }, { "id": "ifeval-prompt-loose", "name": "IFEval prompt-level loose", "kind": "acc", "category": "llm", "description": "IFEval prompt-level loose accuracy (pass@1). Distinct from strict and instruction-level scores.", "url": "https://arxiv.org/html/2501.00656v3" }, { "id": "simple-evals-math", "name": "MATH (simple-evals, original test set, pre-o1 rows)", "kind": "acc", "category": "llm", "url": "https://github.com/openai/simple-evals/blob/652c89d0ca9df547706735883097e9537d40dc47/README.md", "description": "MATH column of the simple-evals README for OpenAI models evaluated before simple-evals switched to MATH-500 (footnote 6 applies to o1 and later). Values first published May-Aug 2024 and unchanged since; zero-shot CoT. OpenAI lab-reported (own models). Distinct from simple-evals-math500." }, { "id": "picovoice-en4-streaming-avg-wer", "name": "Picovoice STT benchmark · English streaming 4-set mean WER", "kind": "err", "category": "stt", "url": "https://github.com/Picovoice/speech-to-text-benchmark/blob/43e7689f013694e9ba9910695643af9c579ca8d2/README.md", "metric": "word_error_rate", "setup": "Picovoice/speech-to-text-benchmark README (Apache-2.0) at 43e7689, section 'English > Streaming Engines Word Error Rate': unweighted mean of LibriSpeech test-clean, LibriSpeech test-other, TED-LIUM and Common Voice (English) WER as printed in the 'Average' column; Picovoice's own normalizer; streaming (chunked, incremental) engines on CPU per engine.py: whisper.cpp via pywhispercpp with 3,000 ms chunks and single_segment=True, Moonshine streaming via moonshine_voice with 100 ms chunks, Vosk via KaldiRecognizer. Same sets and normalizer as picovoice-en4-batch-avg-wer but a different decoding condition (streaming), so a separate series.", "caveats": [ "Streaming WER depends on chunk size and endpointing; whisper.cpp is run in fixed 3 s chunks, which is not how Whisper is normally decoded.", "Picovoice sells the Cheetah streaming engine in the same table; cloud streaming engines are unversioned and not imported." ] }, { "id": "afrispeech200-test108-wer", "name": "AfriSpeech-200 test (108 African accents) · WER", "kind": "err", "category": "stt", "url": "https://arxiv.org/html/2310.00274v1", "metric": "word_error_rate", "setup": "AfriSpeech-200 (Olatunji et al., Intron Health, arXiv:2310.00274v1): 200 h of African-accented English read speech (clinical and general domains, 120 accents, 13 countries). Test split: 750 speakers, 108 accents, 6,346 clips (2,723 general + 3,623 clinical), 18.8 h. Table 4 'Test (108 accents) Both' zero-shot WER of open-source checkpoints run by the authors.", "score_unit": "error ratio (as printed)", "caveats": [ "Transcript normalisation is not specified; the authors' LibriSpeech test-clean WERs for Whisper (0.167) are far above usual values, so Whisper's punctuated output is probably penalised relative to CTC models.", "Same team as AfriSpeech-MultiBench (Intron Health), 2023 runs; a separate protocol from that benchmark." ] }, { "id": "liteasr-esb8-avg-wer", "name": "LiteASR paper · ESB 8-set average WER", "kind": "err", "category": "stt", "url": "https://arxiv.org/html/2502.20583v2", "metric": "word_error_rate", "setup": "LiteASR (Kamahori et al., arXiv:2502.20583v2) Table 1: WER on the eight ESB English test sets (VoxPopuli, AMI, Earnings-22, GigaSpeech, LibriSpeech clean/other, SPGISpeech, TED-LIUM; hf-audio/esb-datasets-test-only-sorted) and their unweighted average, for uncompressed Whisper checkpoints and the authors' low-rank-compressed LiteASR encoders, all run by the authors.", "score_unit": "error ratio (source % / 100)", "caveats": [ "AMI and Earnings-22 WERs are much higher than on the Open ASR Leaderboard for the same checkpoints (different decoding/segmentation); a separate protocol.", "LiteASR rows are the authors' own models (lab); the Whisper baselines are independent." ] }, { "id": "zstts-daps-flores-en-cer", "name": "ZS-TTS-Evaluation · English zero-shot cloning (DAPS speakers, FLORES+) · Whisper-large-v3 CER", "kind": "err", "category": "tts", "url": "https://github.com/Edresson/ZS-TTS-Evaluation/blob/a9086e4192b6ccba9f06c492a5ef68b52ae8ccbe/README.md", "metric": "character_error_rate", "setup": "Edresson Casanova's ZS-TTS-Evaluation (the XTTS paper's comparison code and outputs, MIT): 240 random FLORES+ devtest English sentences synthesised for each of the 20 clean DAPS speakers (10 F / 10 M, one 3-8 s reference clip each); CER from Whisper large-v3 transcripts with punctuation removed (x100 in the file, divided by 100 here). Each system's folder holds its generated audio and metrics.csv.", "score_unit": "error ratio (source % / 100)", "caveats": [ "XTTS and YourTTS are the evaluator's own models (lab).", "Checkpoint files are named only by folder labels; see each result's identity note." ] }, { "id": "zstts-daps-flores-en-ecapa2-secs", "name": "ZS-TTS-Evaluation · English zero-shot cloning (DAPS, FLORES+) · ECAPA2 speaker similarity", "kind": "acc", "category": "tts", "url": "https://github.com/Edresson/ZS-TTS-Evaluation/blob/a9086e4192b6ccba9f06c492a5ef68b52ae8ccbe/README.md", "metric": "speaker_encoder_cosine_similarity", "setup": "Same runs as zstts-daps-flores-en-cer. SECS = cosine similarity between ECAPA2 (Jenthe/ECAPA2) speaker embeddings of the generated clip and the DAPS reference, RMS-normalised to the reference level. Raw cosine.", "score_unit": "cosine similarity (raw)", "caveats": [ "Same audio as the CER series (correlated)." ] }, { "id": "vits2021-ljspeech-mos", "name": "VITS 2021 LJ Speech human MOS", "category": "tts", "kind": "acc", "metric": "human_mos", "raw_scale_min": 1, "raw_scale_max": 5, "url": "https://proceedings.mlr.press/v139/kim21f/kim21f.pdf", "setup": "Original ICML 2021 VITS listening study, naturalness rated 1–5 once per sample; clips amplitude-normalised. Separate 500-utterance LJ Speech and VCTK test splits. Store (MOS − 1) / 4. Do not pool with other MOS studies." }, { "id": "vits2021-vctk-mos", "name": "VITS 2021 VCTK human MOS", "category": "tts", "kind": "acc", "metric": "human_mos", "raw_scale_min": 1, "raw_scale_max": 5, "url": "https://proceedings.mlr.press/v139/kim21f/kim21f.pdf", "setup": "Original ICML 2021 VITS listening study, naturalness rated 1–5 once per sample; clips amplitude-normalised. Separate 500-utterance LJ Speech and VCTK test splits. Store (MOS − 1) / 4. Do not pool with other MOS studies." }, { "id": "evalcrafter2023-visual", "name": "EvalCrafter 2023 visual quality", "category": "video", "kind": "acc", "metric": "human_preference_aligned_composite", "raw_scale_min": 0, "raw_scale_max": 100, "url": "https://arxiv.org/html/2310.11440v3", "setup": "Table 2; 700 annotated prompts. Original 2023 model/API snapshots (Table 1). Composite automatic score learned to align with human preferences, stored /100. Four distinct aspects from the same generations; not direct human ratings, not pass accuracy. Raw constituent metrics deliberately omitted." }, { "id": "evalcrafter2023-alignment", "name": "EvalCrafter 2023 text-video alignment", "category": "video", "kind": "acc", "metric": "human_preference_aligned_composite", "raw_scale_min": 0, "raw_scale_max": 100, "url": "https://arxiv.org/html/2310.11440v3", "setup": "Table 2; 700 annotated prompts. Original 2023 model/API snapshots (Table 1). Composite automatic score learned to align with human preferences, stored /100. Four distinct aspects from the same generations; not direct human ratings, not pass accuracy. Raw constituent metrics deliberately omitted." }, { "id": "evalcrafter2023-motion", "name": "EvalCrafter 2023 motion quality", "category": "video", "kind": "acc", "metric": "human_preference_aligned_composite", "raw_scale_min": 0, "raw_scale_max": 100, "url": "https://arxiv.org/html/2310.11440v3", "setup": "Table 2; 700 annotated prompts. Original 2023 model/API snapshots (Table 1). Composite automatic score learned to align with human preferences, stored /100. Four distinct aspects from the same generations; not direct human ratings, not pass accuracy. Raw constituent metrics deliberately omitted." }, { "id": "evalcrafter2023-temporal", "name": "EvalCrafter 2023 temporal consistency", "category": "video", "kind": "acc", "metric": "human_preference_aligned_composite", "raw_scale_min": 0, "raw_scale_max": 100, "url": "https://arxiv.org/html/2310.11440v3", "setup": "Table 2; 700 annotated prompts. Original 2023 model/API snapshots (Table 1). Composite automatic score learned to align with human preferences, stored /100. Four distinct aspects from the same generations; not direct human ratings, not pass accuracy. Raw constituent metrics deliberately omitted." }, { "id": "fastpitch2020-ljspeech-mos", "name": "FastPitch 2020 LJ Speech human MOS", "category": "tts", "kind": "acc", "metric": "human_mos", "raw_scale_min": 1, "raw_scale_max": 5, "url": "https://github.com/NVIDIA/DeepLearningExamples/blob/729963dd47e7c8bd462ad10bfac7a7b0b604e6dd/PyTorch/SpeechSynthesis/FastPitch/README.md", "setup": "NVIDIA FastPitch original 2020 listening study: first 30 development utterances, at least 250 ratings/model, 60 unique qualified MTurk raters, grapheme input and WaveGlow vocoder. MOS (1–5) stored (MOS−1)/4; separate from VITS/Grad-TTS studies." }, { "id": "fastpitch2021-pairwise-preference", "name": "FastPitch 1.0 vs 1.1 pairwise preference (2021)", "category": "tts", "kind": "acc", "metric": "pairwise_preference", "url": "https://github.com/NVIDIA/DeepLearningExamples/blob/729963dd47e7c8bd462ad10bfac7a7b0b604e6dd/PyTorch/SpeechSynthesis/FastPitch/README.md", "setup": "NVIDIA FastPitch README original 1.0 vs 1.1 pairwise preference study, not MOS; preserve fractions. Only two systems: raw evidence, insufficient models for source calibration." }, { "id": "gradtts2021-ljspeech-mos", "name": "Grad-TTS 2021 LJ Speech human MOS", "category": "tts", "kind": "acc", "metric": "human_mos", "raw_scale_min": 1, "raw_scale_max": 5, "url": "https://proceedings.mlr.press/v139/popov21a/popov21a.pdf", "setup": "Original Table 2 MOS: 40 test sentences, 10 Master MTurk raters per clip, 1–5 scale in 0.5 increments. All systems use HiFi-GAN. Store (MOS−1)/4; not comparable with different MOS studies." }, { "id": "versa-librispeech-test-clean-utmos", "name": "VERSA TTS demo · LibriSpeech test-clean UTMOS", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2412.17667v2", "metric": "predicted_mos_utmos", "setup": "VERSA toolkit TTS demonstration (Appendix C.2): systems synthesise the LibriSpeech test-clean texts with their developers' open-source pipelines and no tuning; zero-shot systems get a random prompt segment from the target speaker, fixed-speaker systems (†) use their own voice; Parler-TTS and CosyVoice get case-recovered text. Metrics computed by the authors with VERSA. UTMOS: UTokyo-SaruLab VoiceMOS 2022 predictor.", "scale_transform": "score = (raw UTMOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "predicted MOS / 5; not human MOS, not accuracy", "setup_sources": [ "https://arxiv.org/html/2412.17667v2" ], "caveats": [ "Automatic MOS prediction, not a listening test.", "Mixes fixed-voice and voice-cloning systems; ground truth scores 4.09.", "Number of utterances not stated." ] }, { "id": "livecodebench-pro-2025q2-easy", "name": "LiveCodeBench Pro 2025Q2 (Easy)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/abs/2506.11928", "description": "LiveCodeBench Pro 2025Q2 release (problems from 2025/04-2025/07, 167 problems in total), Easy difficulty split, pass@1 (avg@8 in the NVIDIA reports). Not LiveCodeBench v5/v6; not the 25Q1 split or the Medium/Hard splits." }, { "id": "livecodebench-pro-2025q2-medium", "name": "LiveCodeBench Pro 2025Q2 (Medium)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/abs/2506.11928", "description": "LiveCodeBench Pro 2025Q2 release (2025/04-2025/07, 167 problems in total), Medium difficulty split, pass@1 (avg@8 in the NVIDIA reports). Same experiment as the Easy split but a disjoint problem subset." }, { "id": "euroeval-en-life-in-the-uk-test-5shot", "name": "EuroEval Life in the UK (English knowledge MC), test split, 5-shot, accuracy", "category": "llm", "kind": "acc", "chance": 0.25, "url": "https://github.com/EuroEval/EuroEval/blob/ee06cd994877150f3b83e044526eb57ea8b10719/src/frontend/md/datasets/english.md#life-in-the-uk", "description": "EuroEval 'life-in-the-uk': 4-option multiple-choice questions scraped from lifeintheuktestweb.co.uk (true/false and too-short/long items removed), 512-question test split, 5 few-shot examples, letter answer. Score = accuracy (EuroEval secondary metric; primary is MCC), mean over 10 iterations." }, { "id": "euroeval-en-life-in-the-uk-val-5shot", "name": "EuroEval Life in the UK (English knowledge MC), validation split, 5-shot, accuracy", "category": "llm", "kind": "acc", "chance": 0.25, "url": "https://github.com/EuroEval/EuroEval/blob/ee06cd994877150f3b83e044526eb57ea8b10719/src/frontend/md/datasets/english.md#life-in-the-uk", "description": "As euroeval-en-life-in-the-uk-test-5shot but EuroEval '(val)' runs on the 256-question validation split." }, { "id": "euroeval-en-hellaswag-test-5shot", "name": "EuroEval HellaSwag (English), test split, 5-shot generative MC, accuracy", "category": "llm", "kind": "acc", "chance": 0.25, "url": "https://github.com/EuroEval/EuroEval/blob/ee06cd994877150f3b83e044526eb57ea8b10719/src/frontend/md/datasets/english.md#hellaswag", "description": "EuroEval 'hellaswag': 2,048-sample test subset of HellaSwag presented as a 4-option multiple-choice question with 5 few-shot examples and a letter answer. Score = accuracy (secondary metric; primary is MCC), mean over 10 iterations." }, { "id": "euroeval-en-hellaswag-val-5shot", "name": "EuroEval HellaSwag (English), validation split, 5-shot generative MC, accuracy", "category": "llm", "kind": "acc", "chance": 0.25, "url": "https://github.com/EuroEval/EuroEval/blob/ee06cd994877150f3b83e044526eb57ea8b10719/src/frontend/md/datasets/english.md#hellaswag", "description": "As euroeval-en-hellaswag-test-5shot but EuroEval '(val)' runs on the 256-item validation split." }, { "id": "pipecat-stt-semantic-wer-pooled", "name": "Pipecat STT Benchmark · pooled semantic WER (1000 smart-turn samples, Sonnet 5.5 judge)", "category": "stt", "kind": "err", "url": "https://github.com/pipecat-ai/stt-benchmark/blob/94f567299a21f8d617039bc69f137a131f5dedd5/README.md", "metric": "semantic_word_error_rate", "setup": "pipecat-ai/stt-benchmark: each service run through a Pipecat real-time (streaming) STT pipeline on the same 1000 samples of pipecat-ai/smart-turn-data-v3.1-train (short conversational voice-agent turns). Reference transcripts generated with Gemini and human-reviewed. 'Semantic WER' judged by Claude Sonnet 5.5 (adaptive thinking, medium effort): punctuation, contractions, filler words, number formats, repeats/restarts ignored; meaning-changing substitutions, hallucinations, missing words, names/numbers/negations counted; split/merged word = one error. 'Pooled WER' = total errors / total reference words (fixed per-reference word count). Samples with no transcript excluded from WER (Transcripts column).", "caveats": [ "LLM-judged metric; the README says rows scored by different judges are not comparable and that differences below ~0.25 pp among the most accurate services are within the 95% bootstrap interval.", "Streaming/realtime endpoints; hosted model aliases can change after the run.", "Rows marked † (previous Sonnet 4.5 judge) are excluded." ] }, { "id": "dailydialog-tts-whisper-large-wer", "name": "DailyDialog TTS · Whisper-large WER (UncovAI deepfake study)", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2601.20510v1", "metric": "word_error_rate", "setup": "Synthetic speech for DailyDialog dialogue transcripts (12,000 samples across three systems); transcribed with OpenAI Whisper-large and scored with jiwer against the original transcripts. Voice cloning from a reference prompt where supported." }, { "id": "fetv-human-video-quality", "name": "FETV manual video quality (mean of static and temporal quality)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2311.01813v3#A6.F20", "task": "text-to-video", "raw_scale_min": 1, "raw_scale_max": 5, "scale_transform": "score = (raw - 1) / 4", "reported_scale": "1-5 Likert", "benchmark_family": "fetv", "setup": { "version": "FETV v3 (arXiv 2311.01813v3), Sec. 4 manual evaluation; values printed in the Figure 20 caption", "metric": "Model mean of static quality and temporal quality, each a 1-5 Likert rating averaged over three human raters; stored (x-1)/4.", "protocol": "619 FETV prompts, one video per prompt per model generated by the authors with official implementations/default settings (CogVideo 33 frames 480x480; ModelScopeT2V 2 s 8 fps 256x256; ZeroScope-v2-576w 3 s 8 fps 576x320).", "limitations": "Small 2023 open-model pool; alignment ratings exist only as figures and are not imported." } }, { "id": "natural-questions", "name": "Natural Questions (closed book)", "kind": "acc", "url": "https://ai.google.com/research/NaturalQuestions" }, { "id": "agieval-gemma-en", "name": "AGIEval English (Gemma report)", "kind": "acc", "url": "https://arxiv.org/html/2403.08295v4#S3.T6" }, { "id": "tulu2-tydiqa-goldp-1shot-f1", "name": "Tulu 2: TyDiQA gold passage, 1-shot F1", "kind": "acc", "url": "https://arxiv.org/html/2311.10702v2" }, { "id": "tulu2-codexeval-pass10", "name": "Tulu 2: CodexEval pass@10", "kind": "acc", "url": "https://arxiv.org/html/2311.10702v2" }, { "id": "arena-agent-ips-202610", "name": "Arena Agent (October 2026 aggregate IPS)", "kind": "linear", "latent_scale": 0.1, "unit": "IPS", "category": "llm", "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/blob/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet", "metric": "inverse-propensity-weighted treatment effect", "description": "Signed net improvement in the evaluated orchestrator/harness. Not Elo, accuracy or a win rate. Store raw IPS; divide by fixed 0.1 (ten percentage points of effect) for the latent fit. Overall aggregate only; component signals are not imported again. October 1 methodology version; highest available published reasoning effort per model in this task. Published intervals and counts retained per observation.", "methodology": "https://arena.ai/blog/agent-arena-methodology" }, { "id": "copen-csj-zero-shot-full-prompt", "name": "COPEN conceptual similarity judgment (zero-shot, full prompt)", "kind": "acc", "category": "llm", "url": "https://aclanthology.org/2022.emnlp-main.335.pdf", "chance": 0.0476 }, { "id": "copen-cpj-zero-shot-full-prompt", "name": "COPEN conceptual property judgment (zero-shot, full prompt)", "kind": "acc", "category": "llm", "url": "https://aclanthology.org/2022.emnlp-main.335.pdf", "chance": 0.5 }, { "id": "copen-cic-zero-shot-full-prompt", "name": "COPEN concept identification in context (zero-shot, full prompt)", "kind": "acc", "category": "llm", "url": "https://aclanthology.org/2022.emnlp-main.335.pdf", "chance": 0.277, "description": "Chance 0.277: published random baseline (https://aclanthology.org/2022.emnlp-main.335.pdf; COPEN (Peng et al., EMNLP 2022), Table 2 'Accuracies (%) of various PLMs on the three tasks using different probing methods. ZP: Zero-shot probing.': row 'Rando)." }, { "id": "mmmu-pro-standard-lmms-exaone45", "name": "MMMU-Pro standard (SqueezeBits lmms-eval)", "kind": "acc", "category": "llm", "url": "https://github.com/SqueezeBits/FlatQuant-EXAONE4.5/blob/5f1ea65605d1f9301cd7d243b1e8a2c6590d2b46/README.md", "chance": 0.128, "description": "Chance 0.128: published random baseline (https://arxiv.org/abs/2409.02813; MMMU-Pro paper, Table 1: 'Random Choice ... 12.8' for Standard (10 Opts).)." }, { "id": "mmmu-pro-vision-lmms-exaone45", "name": "MMMU-Pro vision (SqueezeBits lmms-eval)", "kind": "acc", "category": "llm", "url": "https://github.com/SqueezeBits/FlatQuant-EXAONE4.5/blob/5f1ea65605d1f9301cd7d243b1e8a2c6590d2b46/README.md", "chance": 0.124, "description": "Chance 0.124: published random baseline (https://arxiv.org/abs/2409.02813; MMMU-Pro paper, Table 1: 'Random Choice ... 12.4' for Vision.)." }, { "id": "magpie-heldout-ar-cer", "name": "Magpie held-out Arabic (ar) CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-ar-speaker-similarity", "name": "Magpie held-out Arabic (ar) speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "magpie-heldout-zh-cer", "name": "Magpie held-out Chinese (zh) CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-zh-speaker-similarity", "name": "Magpie held-out Chinese (zh) speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "magpie-heldout-en-cer", "name": "Magpie held-out English (en) CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-en-speaker-similarity", "name": "Magpie held-out English (en) speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "magpie-heldout-en-accented-cer", "name": "Magpie held-out English (en) accented CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-en-accented-speaker-similarity", "name": "Magpie held-out English (en) accented speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "magpie-heldout-fr-cer", "name": "Magpie held-out French (fr) CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-fr-speaker-similarity", "name": "Magpie held-out French (fr) speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "magpie-heldout-de-cer", "name": "Magpie held-out German (de) CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-de-speaker-similarity", "name": "Magpie held-out German (de) speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "magpie-heldout-hi-cer", "name": "Magpie held-out Hindi (hi) CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-hi-speaker-similarity", "name": "Magpie held-out Hindi (hi) speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "magpie-heldout-it-cer", "name": "Magpie held-out Italian (it) CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-it-speaker-similarity", "name": "Magpie held-out Italian (it) speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "magpie-heldout-ja-cer", "name": "Magpie held-out Japanese (ja) CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-ja-speaker-similarity", "name": "Magpie held-out Japanese (ja) speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "magpie-heldout-ko-cer", "name": "Magpie held-out Korean (ko) CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-ko-speaker-similarity", "name": "Magpie held-out Korean (ko) speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "magpie-heldout-pt-cer", "name": "Magpie held-out Portuguese (pt) CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-pt-speaker-similarity", "name": "Magpie held-out Portuguese (pt) speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "magpie-heldout-es-cer", "name": "Magpie held-out Spanish (es) CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-es-speaker-similarity", "name": "Magpie held-out Spanish (es) speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "magpie-heldout-vi-cer", "name": "Magpie held-out Vietnamese (vi) CER", "category": "tts", "kind": "err", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "character_error_rate" }, { "id": "magpie-heldout-vi-speaker-similarity", "name": "Magpie held-out Vietnamese (vi) speaker similarity (pred-context)", "category": "tts", "kind": "acc", "url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m/blob/19806879b16d3f2ccf28fb112b1bcd16a3c7923e/README.md", "metric": "speaker_embedding_cosine" }, { "id": "kandinsky3-reference-text-win", "name": "Human text alignment · wins against Kandinsky 3.0", "kind": "acc", "category": "image", "chance": 0, "url": "https://arxiv.org/html/2312.03511v2", "metric": "outright_win_fraction" }, { "id": "kandinsky3-vs-kandinsky-2-2-text-win", "name": "Kandinsky 3.0 · text alignment wins against kandinsky-2-2", "kind": "acc", "category": "image", "chance": 0, "url": "https://arxiv.org/html/2312.03511v2", "metric": "outright_win_fraction" }, { "id": "kandinsky3-vs-dall-e-3-text-win", "name": "Kandinsky 3.0 · text alignment wins against dall-e-3", "kind": "acc", "category": "image", "chance": 0, "url": "https://arxiv.org/html/2312.03511v2", "metric": "outright_win_fraction" }, { "id": "kandinsky3-vs-sdxl-text-win", "name": "Kandinsky 3.0 · text alignment wins against sdxl", "kind": "acc", "category": "image", "chance": 0, "url": "https://arxiv.org/html/2312.03511v2", "metric": "outright_win_fraction" }, { "id": "kandinsky3-reference-visual-win", "name": "Human visual quality · wins against Kandinsky 3.0", "kind": "acc", "category": "image", "chance": 0, "url": "https://arxiv.org/html/2312.03511v2", "metric": "outright_win_fraction" }, { "id": "kandinsky3-vs-kandinsky-2-2-visual-win", "name": "Kandinsky 3.0 · visual quality wins against kandinsky-2-2", "kind": "acc", "category": "image", "chance": 0, "url": "https://arxiv.org/html/2312.03511v2", "metric": "outright_win_fraction" }, { "id": "kandinsky3-vs-dall-e-3-visual-win", "name": "Kandinsky 3.0 · visual quality wins against dall-e-3", "kind": "acc", "category": "image", "chance": 0, "url": "https://arxiv.org/html/2312.03511v2", "metric": "outright_win_fraction" }, { "id": "kandinsky3-vs-sdxl-visual-win", "name": "Kandinsky 3.0 · visual quality wins against sdxl", "kind": "acc", "category": "image", "chance": 0, "url": "https://arxiv.org/html/2312.03511v2", "metric": "outright_win_fraction" }, { "id": "coco30k-fid-kandinsky21", "name": "COCO-30k zero-shot FID ↓ (Kandinsky 2.1 protocol)", "category": "image", "kind": "linear", "direction": "lower", "latent_scale": 10, "unit": "FID", "metric": "frechet_inception_distance", "url": "https://huggingface.co/kandinsky-community/kandinsky-2-1/blob/04b366659663bf60deacb96fd3e38f1cb39a3be4/README.md" }, { "id": "frontiermath-erdos", "name": "FrontierMath: Erdős problems", "kind": "acc", "chance": 0, "category": "llm", "url": "https://epoch.ai/benchmarks/use-this-data", "description": "Epoch AI internal evaluation file frontiermath_erdos.csv ('FrontierMath-Erdos' in Epoch's benchmark metadata, released 2026-09-03). Published 'Best score (across scorers)'. Few models run so far." }, { "id": "matharena-arxivmath-2026-08", "name": "MathArena ArXivMath August 2026, with tools (57 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/arxivmath-0826", "family": "matharena-arxivmath-tools", "description": "Research-level final-answer questions from August 2026 arXiv papers. Unlike earlier months, models run in an agent harness with shell, Python and SageMath (no internet; 12 h / $100 per attempt) and answers are graded by MathArena's LLM answer judge. Score = mean correctness over attempts, per problem then over problems. Not merged with the earlier no-tools months." }, { "id": "matharena-brokenarxiv-2026-08", "name": "MathArena BrokenArXiv August 2026, with tools (56 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/brokenarxiv-0826", "family": "matharena-brokenarxiv-tools", "description": "Model is asked to prove a subtly false statement derived from an August 2026 arXiv theorem, in an agent harness with shell, Python and SageMath (no internet). Judged by Gemini 3.8 Flash with MathArena's arxiv judge; 1 = explicitly recognises the statement is false. Score = mean normalised judge score. Not merged with earlier no-tools months." }, { "id": "matharena-arxivlean-2026-06", "name": "MathArena ArXivLean June 2026 (46 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://huggingface.co/datasets/MathArena/arxivlean-0626", "family": "matharena-arxivlean", "description": "Prove formal Lean 4 (v4.31.0, mathlib) statements from June 2026 arXiv papers with MathArena's Lean tool loop (max 200 tool calls; harness opt-in per model config); success = Lean-verified proof of the unmodified statement. One attempt per problem." }, { "id": "automationbench-public-zapier", "name": "AutomationBench public set (Zapier README)", "kind": "acc", "category": "llm", "url": "https://github.com/zapier/AutomationBench", "description": "Pass rate on AutomationBench's 600-task public set as published in the Zapier README (maintainer runs, not the private-set leaderboard). Package version per row not stated, so kept separate from automationbench, automationbench-public and automationbench-v1-0-6." }, { "id": "omniasr-paper-fleurs81-test-cer", "name": "Omnilingual ASR paper · FLEURS-81 test mean CER (Whisper-covered languages)", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2511.09690v1", "metric": "character_error_rate", "setup": "Omnilingual ASR team, arXiv:2511.09690v1, Table 5: character error rate on the FLEURS test split, averaged (unweighted) across the 81 FLEURS languages that Whisper supports; all systems run by the authors, no LM fusion for the recorded rows. CER % / 100.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Language subset is fixed to the languages Whisper covers; the body text calls the sets 'MMS-Lab-63, FLEURS-82' while the table header says MMS-Lab-66 / FLEURS-81 (header recorded).", "Omnilingual rows are the authors' own models (lab); Whisper rows are run by the same authors (independent of OpenAI).", "Average over languages, so Whisper's long-tail failures dominate its mean." ] }, { "id": "omniasr-paper-cv22-76-test-cer", "name": "Omnilingual ASR paper · Common Voice 22 (76 languages) test mean CER", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2511.09690v1", "metric": "character_error_rate", "setup": "Omnilingual ASR team, arXiv:2511.09690v1, Table 5: character error rate on the Common Voice 22 test split, averaged (unweighted) across the 76 CV22 languages that Whisper supports; all systems run by the authors, no LM fusion. CER % / 100; can exceed 1 for Whisper small.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Language subset is fixed to the languages Whisper covers; the body text calls the sets 'MMS-Lab-63, FLEURS-82' while the table header says MMS-Lab-66 / FLEURS-81 (header recorded).", "Omnilingual rows are the authors' own models (lab); Whisper rows are run by the same authors (independent of OpenAI).", "Average over languages, so Whisper's long-tail failures dominate its mean.", "Whisper small CER exceeds 100% (hallucination/insertions)." ] }, { "id": "distil-whisper-ood4-shortform-wer", "name": "Distil-Whisper · 4-set out-of-distribution short-form mean WER", "category": "stt", "kind": "err", "url": "https://github.com/huggingface/distil-whisper/blob/a66f5c85be1e32c504e8ed68df7802d2b96db4c0/README.md", "metric": "word_error_rate", "setup": "Hugging Face Distil-Whisper evaluation (README headline table; protocol of Gandhi et al., arXiv:2311.00430, Tables 4-5): unweighted mean WER over four short-form English test sets unseen in Distil-Whisper training: CHiME-4, Earnings-22, FLEURS (en) and SPGISpeech; Whisper English normaliser; short-form (<30 s) greedy decoding. WER % / 100.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Distil rows are Hugging Face's own models (lab); the Whisper large-v3 row is Hugging Face's run of OpenAI's model.", "Per-set values for the newer rows (large-v3, distil-large-v3, distil-small.en) are not published in the README; protocol inferred from the identical older rows." ] }, { "id": "mls-fr-wer", "name": "Multilingual LibriSpeech French test WER", "kind": "err", "category": "stt", "description": "Word error rate on MLS French test (fraction). Text normalisation, segmentation and decoding follow the publishing source and are recorded per result. Separate evaluator keys per source.", "url": "https://www.openslr.org/94" }, { "id": "mls-de-wer", "name": "Multilingual LibriSpeech German test WER", "kind": "err", "category": "stt", "description": "Word error rate on MLS German test (fraction). Text normalisation, segmentation and decoding follow the publishing source and are recorded per result. Separate evaluator keys per source.", "url": "https://www.openslr.org/94" }, { "id": "rvcbench-libritts-whisper-medium-wer", "name": "RVCBench LibriTTS clean voice cloning · Whisper-medium WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2602.00443v2", "metric": "word_error_rate", "setup": "Same RVCBench clean English voice-cloning condition as rvcbench-libritts-ecapa-sim (speaker-balanced LibriTTS subset, re-paired reference/target text, 4k utterances, default inference by the authors). WER of Whisper-medium transcripts (no timestamps, no previous-text conditioning) against the target text after lowercasing and punctuation removal; mean with 1000-sample bootstrap (Appendix C.1, Table 13).", "score_unit": "error ratio as printed (two decimals)", "caveats": [ "Printed to two decimals only (0.05 = 5%), so small differences are not resolved.", "Robustness benchmark baseline condition, not a curated TTS test set." ] }, { "id": "versa-librispeech-test-clean-whisper-wer", "name": "VERSA TTS demo · LibriSpeech test-clean Whisper WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2412.17667v2", "metric": "word_error_rate", "setup": "Same VERSA TTS demonstration as versa-librispeech-test-clean-utmos (Appendix C.2, Table 6): systems synthesise the LibriSpeech test-clean texts with their developers' pipelines, no tuning; WER of Whisper transcripts against the input text computed with VERSA's whisper_wer metric. WER % / 100.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Whisper size not printed in the paper; VERSA's whisper_wer defaults to the 'large' tag.", "Mixes fixed-voice and zero-shot cloning systems; ground truth scores 2.59%." ] }, { "id": "aider-code-editing", "name": "Aider code editing (133 Exercism Python exercises, two attempts)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/docs/leaderboards/edit.md", "description": "Aider's original code editing benchmark (superseded by the polyglot leaderboard): the model edits Python source files to complete 133 small Exercism exercises and must apply its own edits; score is pass_rate_2, the percent of exercises whose tests pass after at most two attempts (the second sees the test errors). Edit format per run in the note. Distinct from aider-polyglot (225 exercises, six languages)." }, { "id": "wildbench-v2-wb-score-macro", "name": "WildBench v2 WB-Score (task macro)", "kind": "linear", "latent_scale": 10, "unit": "WB-Score", "category": "llm", "url": "https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/leaderboard/data_dir/all_stat_wildbench.-1.json", "metric": "GPT-4o checklist score rescaled as (score-5)x2x10, macro-averaged over 5 task categories", "description": "WildBench v2 (1,024 challenging real-user tasks) scored by GPT-4o with per-task checklists; each 1-10 score is rescaled as (score-5)x2 and multiplied by 10 (0 = a 5/10 answer), then macro-averaged over Creative, Planning & Reasoning, Math & Data Analysis, Information/Advice seeking and Coding & Debugging ('WB_score.task_macro'). Signed raw units, not accuracy; fixed latent_scale 10 (ten WB-Score points). Not WB-Elo or pairwise reward." }, { "id": "rapidata-i2v-realism-overall", "name": "Rapidata image-to-video realism (overall Elo)", "category": "video", "kind": "elo", "url": "https://huggingface.co/datasets/Rapidata/image-to-video-realism", "task": "image-to-video", "setup": { "url": "https://huggingface.co/datasets/Rapidata/image-to-video-realism/blob/d7abf89e71cb45fff0b08d70df389026cb78024f/README.md", "version": "Rapidata/image-to-video-realism dataset card @ d7abf89 (2026-10-01): 19 participants incl. a real-footage baseline, 25 prompts with image assets, 2,981 comparisons, 522,210 human votes", "metric": "Published 'Overall ranking' ELO over the two human leaderboards (Photorealism: 'Which video looks more like it was shot by a real camera?'; Visual integrity: 'Which video has more glitches and is more likely to be AI generated?'), prompt not shown to annotators.", "protocol": "Head-to-head comparisons of two clips generated from the same prompt image and text; userScore-weighted crowd votes collected with the Rapidata SDK.", "limitations": "Generation settings (resolution, duration, API endpoint) are not documented on the card; only 25 prompts. Ratings are relative to this participant pool (includes a ground-truth baseline at ~1470) and are not on Arena or other Rapidata scales. Do not mix with the Photorealism / Visual integrity sub-rankings." }, "reported_scale": "Elo", "benchmark_family": "rapidata-i2v-realism" }, { "id": "rapidata-relative-camera-i2v", "name": "Rapidata relative camera movement (I2V, alignment Elo)", "category": "video", "kind": "elo", "url": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k", "task": "image-to-video", "setup": { "url": "https://huggingface.co/datasets/Rapidata/relative-camera-movement-human-preference-900k/blob/e03ac787593bada12ac6b088d3d35da4716517c7/README.md", "version": "Rapidata relative-camera-movement dataset card @ e03ac78 (2026-09-29): 15 models, 50 stills, 220 hand-written scene-specific camera prompts (camera movement and perspective switching), 3,270 clips", "metric": "Published Bradley-Terry style ELO on the Alignment leaderboard ('Which video matches the description better?', instruction shown).", "protocol": "Each still + prompt sent to every model as an image-to-video request (still as first frame); 22-44 votes per pair (39.7 avg) from a dedicated Rapidata audience in 125 countries.", "limitations": "Per-model resolution/duration not documented on this card. Ratings are relative to this pool and not comparable with the scene-agnostic Rapidata camera-movement Elo." }, "reported_scale": "Elo", "benchmark_family": "rapidata-relative-camera-i2v" }, { "id": "heim-alignment-winrate", "name": "HEIM text-image alignment (aspect win rate, v1.1.0)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2311.04287v1#S7.T5", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2311.04287v1#S7.T5", "version": "HEIM v1.1.0 as summarised in arXiv 2311.04287v1 Table 5 (26-model pool)", "metric": "Alignment aspect win rate: probability that the model outperforms another of the 26 models chosen uniformly at random, head-to-head per metric (human-rated alignment and CLIPScore) over the alignment scenarios; already 0-1.", "protocol": "Images generated by the HEIM authors with each model; five Amazon Mechanical Turk annotators per human-rated sample; scenarios MS-COCO 2014 val (incl. art-style variants), CUB-200-2011, DrawBench, PartiPrompts and others listed in Table 2.", "limitations": "Relative to the fixed 26-model HEIM pool (adding/removing models changes win rates); mixes human and CLIPScore metrics; 2023 checkpoints and API versions. Other aspect columns (quality, aesthetics, etc.) are not imported." }, "reported_scale": "0-1 win rate", "benchmark_family": "heim" }, { "id": "imagereward-human-wins", "name": "ImageReward human ranking of T2I models (#wins, real user prompts)", "category": "image", "kind": "linear", "latent_scale": 100, "direction": "higher", "unit": "wins", "url": "https://arxiv.org/html/2304.05977v4#S2.T1", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2304.05977v4#S2.T1", "version": "ImageReward report v4 (28 Dec 2023), Table 1, Real User Prompts / Human Eval. #Win; fixed pool of 6 models", "metric": "Aggregated human win count of each model against the other five models (raw count, higher is better); stored unscaled.", "protocol": "Researcher annotation by the authors: 100 real-user (DiffusionDB) test prompts; each model generates 10 outputs per prompt, the best of 10 is picked, annotators rank the six models' images per prompt; #Win = aggregated win count of the model against all others (Sec. 2.3)", "limitations": "Counts depend on the six-model pool and the number of annotations (denominator not published), so only comparable within this run; best-of-10 selection per model; early-2023 checkpoints/API." }, "reported_scale": "win count", "benchmark_family": "imagereward" }, { "id": "imagereward-coco30k-fid", "name": "MS-COCO 2014 zero-shot FID-30k at 256px ↓ (ImageReward protocol)", "category": "image", "kind": "linear", "direction": "lower", "latent_scale": 10, "unit": "FID", "metric": "frechet_inception_distance", "url": "https://arxiv.org/html/2304.05977v4#S2.T1", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2304.05977v4#S2.T1", "version": "ImageReward report v4 (28 Dec 2023), Table 1, MS-COCO 2014 'Zero-shot FID' column, author-computed rows only", "metric": "FID between 30k generated images and MS-COCO 2014 validation images, both at 256x256; lower is better.", "protocol": "Zero-shot FID on 30k MS-COCO 2014 validation captions, outputs unified to 256x256, optimal classifier-free guidance chosen by grid search over [1.5, 2.0, 3.0, 4.0, 5.0] (Sec. 2.3, Table 1 caption); computed by the ImageReward authors", "limitations": "FID correlates poorly with human preference for modern models (Spearman 0.09 to human ranking in the same table); CFG grid-searched per model; not comparable with other papers' COCO FID numbers (different resolution/guidance/reference sets)." }, "reported_scale": "FID", "benchmark_family": "coco-fid" }, { "id": "vvrbench-accuracy", "name": "VVRBench accuracy (10,000 verifiable synthetic-scene tasks)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2609.35641v1#S3.T3", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2609.35641v1#S3", "version": "VVRBench main split as released with arXiv:2609.35641v1 (28 Sep 2026): 10,000 tasks, complexity 3-48, 32 constraint types", "metric": "Task accuracy: share of tasks whose generated image passes every constraint (counts, attributes, sizes, spatial relations) under the deterministic, reference-free pixel-to-object verifier; reported in %, stored /100.", "protocol": "VVRBench main split, 10,000 programmatically generated tasks of structural complexity 3-48 over 32 constraint types; a generated image succeeds only if every constraint passes the deterministic pixel-to-object verifier; accuracy = mean success over tasks. Open-weight models at Table 9 settings (steps/CFG); API models at medium quality 1024x1024; responses without an image count as failures.", "limitations": "Synthetic geometric-object scenes only (precise instruction following, not aesthetics). Do not pool with VVRBench-Fast (820-task subset) or VVRBench-Challenge. Near-zero scores for weak models are at the floor." }, "reported_scale": "% accuracy", "benchmark_family": "vvrbench" }, { "id": "vvrbench-challenge-accuracy", "name": "VVRBench-Challenge accuracy (720 harder verifiable tasks)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2609.35641v1#S3.F2.fig2", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2609.35641v1#S3", "version": "VVRBench-Challenge as released with arXiv:2609.35641v1: 720 tasks, complexity 45-80 (20 per integer complexity), 14 additional group-level constraint types", "metric": "Task accuracy: share of tasks whose image passes every constraint under the deterministic verifier; %, stored /100.", "protocol": "VVRBench-Challenge, 720 tasks of complexity 45-80 (20 per integer complexity) adding 14 group-level constraint types; same deterministic verifier and all-constraints-pass accuracy. API models only: GPT models at medium quality 1024x1024, Gemini models at 1K (Table 9); responses without an image count as failures.", "limitations": "API models only in this report; best model 21.4%, so low scores are compressed. Separate task set from the main VVRBench split; not pooled with it." }, "reported_scale": "% accuracy", "benchmark_family": "vvrbench" }, { "id": "chartsync-overall", "name": "ChartSync chart editing (Overall)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2607.10301v1#S4.T2", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2607.10301v1#S3", "version": "ChartSync as released with arXiv:2607.10301v1 (11 Jul 2026): 870 triplets, Table 2", "metric": "Overall = equal average of OCR F1, SSIM, TESR, VLCS and BFS (each 0-100), stored /100. VLM-judged parts use Gemini-3.1-Pro at temperature 0.1.", "protocol": "ChartSync: 870 expert-validated chart-editing triplets (9 chart categories, 4 task types, 235 geometry-coupled VLCE instances); each model edits the source chart image per instruction; Overall = equal average of five metric-level scores: OCR F1 and SSIM against the programmatic ground truth, and Gemini-3.1-Pro judged TESR (text edit success), VLCS (visuo-logical consistency, VLCE subset only) and BFS (background fidelity).", "limitations": "Charts only (statistical-chart text and geometry edits). Mixes pixel metrics with a single VLM judge; VLCS covers only the 235 VLCE items. Metric-level summary, not a per-sample average." }, "reported_scale": "0-100", "benchmark_family": "chartsync" }, { "id": "infoedit-avg-sr", "name": "InfoEdit infographic editing (average success rate)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2609.33286v1#S3.T2", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2609.33286v1#S3", "version": "InfoEdit as released with arXiv:2609.33286v1 (27 Sep 2026): 1,000 infographics x 4 edit tasks", "metric": "Avg. = mean over the four tasks of success rate (edit compliance AND content preservation, judged by Gemini-3.1-Pro); %, stored /100.", "protocol": "InfoEdit: 1,000 rendered infographics (eight logical-relation families) x four reflow-requiring edit tasks (Expand-Text, Insert-Element, Swap-Block, Reshape-Canvas) = 4,000 edits; a sample succeeds (SR) only if both edit compliance and content preservation pass under the Gemini-3.1-Pro judge (validated against human majority votes, Appendix E); Avg. = mean SR over the four tasks.", "limitations": "Infographic layouts only; single MLLM judge; most models are near the floor (0-5%), so the run mainly separates frontier models." }, "reported_scale": "% success", "benchmark_family": "infoedit" }, { "id": "paintbench-miou", "name": "PaintBench precise editing (mIoU, 20-task average)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2606.00188v1#S4.T1", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2606.00188v1#S3", "version": "PaintBench as released with arXiv:2606.00188v1 (29 May 2026), baseline visual conditions, Table 1", "metric": "Benchmark Avg.: macro-average over 20 tasks of mean IoU (%) between the model's edit and the deterministic ground truth; stored /100.", "protocol": "PaintBench: procedurally generated single-answer editing problems (20 task types in four categories: geometric transformation, structural manipulation, color change, symbolic reasoning) at 1024x1024 baseline conditions; deterministic pixel-level scoring of the edited image against the programmatic ground truth (mean IoU over edit classes), no judge model; Benchmark Avg. = macro-average over the 20 tasks.", "limitations": "Synthetic shapes/colours on plain backgrounds; precise single-answer edits rather than open-ended editing quality; best model 17.1%, most near the floor." }, "reported_scale": "% mIoU", "benchmark_family": "paintbench" }, { "id": "mechverse-wmb-sum", "name": "MechVerse mechanical-motion I2V (WorldModelBench-judge sum)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2605.14843v1#S4.T1", "task": "image-to-video", "setup": { "url": "https://arxiv.org/html/2605.14843v1#S4", "version": "MechVerse test split as released with arXiv:2605.14843v1 (14 May 2026), Table 1 WorldModelBench columns", "metric": "Sum of WorldModelBench judge scores (instruction 0-3, physics 0-5, common sense 0-2), out of 10; stored /10.", "protocol": "MechVerse test split: image-to-video generation of mechanical assemblies (gears, linkages, cam-follower, pistons, multi-part mechanisms) from a first frame plus a structured motion prompt; default inference settings per model; outputs scored with the WorldModelBench MLLM judge: instruction following (0-3) + physics (0-5) + common sense (0-2) = Sum (0-10).", "limitations": "MLLM judge; the authors report that judge scores rise with mechanism complexity while human-rated motion correctness falls, so the judge is lenient on part-level kinematic errors. VBench-I2V columns in the same table are separate automatic metrics and are not imported." }, "reported_scale": "0-10", "benchmark_family": "mechverse" }, { "id": "rapidata-benchai-image-overall", "name": "Rapidata Benchmark.ai image benchmark (overall Elo)", "category": "image", "kind": "elo", "url": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M", "task": "text-to-image", "setup": { "url": "https://huggingface.co/datasets/Rapidata/text-to-image-human-preference-6M/blob/2f602f465ad418ac886f2fde92ba42b85e1369db/README.md", "version": "Rapidata/text-to-image-human-preference-6M dataset card @ 2f602f4 (2026-10-01): 43 models, 1,514 prompts, 680,548 comparisons, 6,123,853 votes", "metric": "Published 'Overall ranking (ELO, aggregated across all four leaderboards)': Alignment, Preference, Coherence and Text coherence human pairwise leaderboards combined.", "protocol": "Benchmark.ai Image Benchmark run by Rapidata: 43 text-to-image models, 1,514 prompts (616 asking for rendered text), 680,548 head-to-head image pairs for the same prompt; 6.1M human votes on four questions (Alignment with prompt shown; Preference, Coherence and Text coherence without the prompt); Overall = Rapidata's published ELO aggregated across the four leaderboards. Generation settings per model are not documented on the card.", "limitations": "Per-model generation settings (quality tier, resolution, endpoint) are not documented on the card except where encoded in the model label; ratings are relative to this 43-model pool and not on the LMArena or other Rapidata scales. Do not import the four component leaderboards separately (same comparisons)." }, "reported_scale": "Elo", "benchmark_family": "rapidata-benchai-image" }, { "id": "rapidata-face-alignment", "name": "Rapidata human-face generation alignment (Elo)", "category": "image", "kind": "elo", "url": "https://huggingface.co/datasets/Rapidata/Face_Generation_Benchmark", "task": "text-to-image", "setup": { "url": "https://huggingface.co/datasets/Rapidata/Face_Generation_Benchmark/blob/c6ed7eab14620db5daa77aec0feda5992091f5b3/README.md", "version": "Rapidata/Face_Generation_Benchmark card @ c6ed7ea: 12 models, ~22,000 responses (benchmark run August/September 2025)", "metric": "Published leaderboard 'score' (Elo-style rating from pairwise wins) for prompt alignment on descriptions of people.", "protocol": "Rapidata Human Face Generation Alignment: 12 text-to-image models (Replicate model ids) generate images from descriptions of people; ~22,000 human pairwise responses to 'Which Image follows the description of the human better?'; published leaderboard score (Elo-style rating) with wins and total matches.", "limitations": "Faces/people prompts only; alignment question only (no aesthetics); prompt count and generation settings beyond the Replicate model id are not documented; ratings relative to this 12-model pool." }, "reported_scale": "Elo", "benchmark_family": "rapidata-face-alignment" }, { "id": "videoargus-t2v", "name": "VideoArgus-Bench T2V (rubric-grounded score)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2608.05485v1#S3.T2", "task": "text-to-video", "setup": { "url": "https://arxiv.org/html/2608.05485v1#S4", "version": "VideoArgus-Bench as released with arXiv:2608.05485v1 (6 Aug 2026), Table 2 T2V leaderboard", "metric": "Mean VideoArgus score (0-10, importance-weighted criterion scores with hard-criterion caps), stored /10.", "protocol": "VideoArgus-Bench (1,026 human-verified input instances over five tasks); every candidate video for an input is scored against the same frozen, output-blind, sample-specific rubric (generated once with Claude Opus 4.8; 6-17 criteria with importance weights and hard-criterion caps), with criterion-level evidence and judgments by Qwen3.6-27B plus visual tools; VideoArgus score 0-10 per video, averaged per model and task.", "limitations": "Automatic rubric + VLM judge (Qwen3.6-27B), validated against 15 annotators on a separate 210-input set; one leaderboard per task, the other three tasks (TS2V, TV2V, TSV2V) are not imported. Generation settings per model not documented." }, "reported_scale": "0-10", "benchmark_family": "videoargus" }, { "id": "videoargus-ti2v", "name": "VideoArgus-Bench TI2V (rubric-grounded score)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2608.05485v1#S3.T2", "task": "image-to-video", "setup": { "url": "https://arxiv.org/html/2608.05485v1#S4", "version": "VideoArgus-Bench as released with arXiv:2608.05485v1 (6 Aug 2026), Table 2 TI2V leaderboard", "metric": "Mean VideoArgus score (0-10, importance-weighted criterion scores with hard-criterion caps), stored /10.", "protocol": "VideoArgus-Bench (1,026 human-verified input instances over five tasks); every candidate video for an input is scored against the same frozen, output-blind, sample-specific rubric (generated once with Claude Opus 4.8; 6-17 criteria with importance weights and hard-criterion caps), with criterion-level evidence and judgments by Qwen3.6-27B plus visual tools; VideoArgus score 0-10 per video, averaged per model and task.", "limitations": "Automatic rubric + VLM judge (Qwen3.6-27B), validated against 15 annotators on a separate 210-input set; one leaderboard per task, the other three tasks (TS2V, TV2V, TSV2V) are not imported. Generation settings per model not documented." }, "reported_scale": "0-10", "benchmark_family": "videoargus" }, { "id": "datapoint-image-bench-elo", "name": "Datapoint Image Bench (overall Elo)", "category": "image", "kind": "elo", "url": "https://huggingface.co/datasets/datapointai/text-2-image-human-preferences-2m", "task": "text-to-image", "setup": { "url": "https://huggingface.co/datasets/datapointai/text-2-image-human-preferences-2m/blob/e1d8719a2d521eac6c62ee84f329afc2c03ec928/README.md", "version": "Datapoint Image Bench overall board published 2026-08-19 (30 models, 500 prompts), as shown in the datapointai/text-2-image-human-preferences-2m card @ e1d8719", "metric": "Overall Elo (Bradley-Terry on raw votes, equal weight per category, anchored FLUX.1 [schnell] = 1000), whole points.", "protocol": "Datapoint Image Bench: 30 text-to-image models, 500 stress-test prompts in 10 categories (counts, typography, negation, ordering, composition), one image per prompt with a single fixed seed, prompts passed verbatim (no rewriting, no best-of-N); complete round-robin, every pair judged blind by 10 validated annotators ('Which image do you prefer?'), 2,161,160 votes; Elo from a Bradley-Terry fit on raw votes with equal category weight, anchored at FLUX.1 [schnell] = 1000, rounded to whole points.", "limitations": "Values are printed only in the card's chart image (the vote data is gated); single seed per prompt; ratings relative to this 30-model pool. Category boards are not imported." }, "reported_scale": "Elo", "benchmark_family": "datapoint-image-bench" }, { "id": "datapoint-video-bench-elo", "name": "Datapoint Video Bench (overall Elo)", "category": "video", "kind": "elo", "url": "https://huggingface.co/datasets/datapointai/text-to-video-human-preferences-326k", "task": "text-to-video", "setup": { "url": "https://huggingface.co/datasets/datapointai/text-to-video-human-preferences-326k/blob/49295b5d625c04480c92c33a2b4c502bf571110b/README.md", "version": "Datapoint Video Bench overall board in the datapointai/text-to-video-human-preferences-326k card @ 49295b5 (2026-09-28): 15 models, 314 prompts", "metric": "Overall Elo (Bradley-Terry on raw votes, equal weight per category, 14 original models centred at 1000); 95% rank spread kept in notes.", "protocol": "Datapoint Video Bench: 15 text-to-video models, 314 prompts in eight categories (camera moves, physics/causality, object permanence, multi-subject interaction, human motion, legible text, stylized animation, product ads); one 1280x720 clip per model and prompt, audio stripped; full round-robin with 10 blind votes per pair ('Which video better fulfills the prompt with the highest overall quality?'), 325,520 votes; Elo from a Bradley-Terry fit on raw votes, equal weight per category, centred so the 14 original models average 1000.", "limitations": "Audio removed, so audio-capable models are judged on video only; one clip per model and prompt; P-Video-2-Pro added in a second round against 13 models. Ratings relative to this 15-model pool." }, "reported_scale": "Elo", "benchmark_family": "datapoint-video-bench" }, { "id": "evalalign-human-alignment", "name": "EvalAlign human text-image alignment score", "category": "image", "kind": "linear", "direction": "higher", "latent_scale": 1, "unit": "mean human alignment score", "url": "https://arxiv.org/html/2406.16562v3#S5.T3", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2406.16562v3#S5", "version": "EvalAlign report v3 (10 Oct 2024), Table 3 'Human' column; fixed 24-model pool, 100 prompts", "metric": "Mean human text-image alignment score per model from the EvalAlign question set (raw units; observed range 4.15-5.45), stored unscaled.", "protocol": "EvalAlign test set: 24 text-to-image models each generate images for the same 100 prompts (2.4k images); human annotators answer the text-image alignment question set (object, count, colour, style, spatial and other skills) with fixed scoring options; 'Human' = mean human alignment score per model (raw units, higher is better).", "limitations": "Only 100 prompts per model; absolute scale depends on the question set, so only comparable within this run. The faithfulness table (Table 2) and the automatic metric columns (EvalAlign, HPS v2, CLIPScore, ImageReward, PickScore) are not imported." }, "reported_scale": "raw human score", "benchmark_family": "evalalign" }, { "id": "coco10k-fid-sdxl-lightning", "name": "MS-COCO 10K FID (whole image) ↓ (SDXL-Lightning protocol)", "category": "image", "kind": "linear", "direction": "lower", "latent_scale": 10, "unit": "FID", "metric": "frechet_inception_distance", "url": "https://arxiv.org/html/2402.13929v3#S4.T2", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2402.13929v3#S4", "version": "SDXL-Lightning report v3 (2 Mar 2024), Table 2 'FID (Whole)' column", "metric": "FID between images for the first 10K COCO validation captions and the matching COCO images, whole image resized to 299px; lower is better.", "protocol": "Authors' FID on the first 10K MS-COCO validation captions: images generated by each model at its listed step count (1024px class models), FID-Whole = standard FID with whole images resized to 299px for InceptionV3 against the matching COCO ground-truth images; lower is better.", "limitations": "FID tracks distribution match more than prompt following or preference; step count per model is part of the setting; not comparable with other papers' COCO FID (different subsets/resolutions). FID-Patch and CLIP columns not imported." }, "reported_scale": "FID", "benchmark_family": "coco-fid" }, { "id": "unigenbench-phys-all-cosmos3", "name": "UniGenBench + Physical-AI prompts, All 1,170 (Cosmos 3 report)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2606.02800v4#S6.T11", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2606.02800v4#S6", "version": "Cosmos 3 report v4 (23 Jun 2026), Table 11 'UniGenBench All (1170)': original 600 UniGenBench prompts plus NVIDIA's 570 UniGenBench-Phys prompts", "metric": "Mean testpoint accuracy (binary pass/fail per testpoint, Gemini 3.1 Pro judge) over all 1,170 prompts; %, stored /100.", "protocol": "UniGenBench (600 prompts, 5 themes/20 subthemes, 10 primary and 27 sub-criteria) augmented by NVIDIA with 570 Physical-AI prompts (UniGenBench-Phys); Gemini 3.1 Pro judges each generated image against each testpoint (binary pass/fail); score = mean testpoint accuracy over all 1,170 prompts ('All'). Cosmos 3 images generated from upsampled JSON prompts (shift 3.0, guidance 4.0, 50 steps, Table 21).", "limitations": "NVIDIA-specific variant of UniGenBench (added Physical-AI subset, Gemini 3.1 Pro judge), not the original UniGenBench leaderboard; NVIDIA's own model is lab-reported. The report's Table 1 overview prints 90.85 for Gemini 3 Pro Image while Table 11 prints 90.69 (Table 11 used)." }, "reported_scale": "% testpoint accuracy", "benchmark_family": "unigenbench" }, { "id": "mindworldbench-intention-accuracy", "name": "MindWorldBench intention accuracy (I2V mental-state reasoning)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2609.39147v1#S3.T1", "task": "image-to-video", "setup": { "url": "https://arxiv.org/html/2609.39147v1#S3", "version": "MindWorldBench as released with arXiv:2609.39147v1 (30 Sep 2026; ACM MM 2026), 744 prompts, Table 1", "metric": "Intention Accuracy: % of prompts whose generated action matches the intention implied by the mental-state condition (binary, Gemini 3.1 Pro judge); stored /100.", "protocol": "MindWorldBench: 744 expert-designed image-to-video prompts in which a character's mental state (belief, desire, perception) should determine one of two candidate actions; each I2V system generates one video per prompt; Gemini 3.1 Pro judges with a structured multi-step rubric pipeline (validated on 100 videos against 10 human experts); Intention Accuracy = % of prompts whose generated action follows the intention implied by the mental-state condition.", "limitations": "Single VLM judge; the benchmark's other columns (visual quality, commonsense plausibility 1-5; world-state maintenance %) are not imported. Two candidate actions per prompt, so a model ignoring the mental state can still hit the intended action by chance." }, "reported_scale": "% accuracy", "benchmark_family": "mindworldbench" }, { "id": "vigor-bench-result-avg-image", "name": "ViGoR-Bench zero-shot visual reasoning, image models (Result Avg)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2603.25823v1#S3.T3", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2603.25823v1#S3", "version": "ViGoR-Bench as released with arXiv:2603.25823v1 (26 Mar 2026), Table 3 'Result Metric (%) Avg'", "metric": "Mean of the four Result metrics (BC, RO, VQ, RS), Gemini-2.5-Pro judge with ground-truth references; %, stored /100.", "protocol": "ViGoR-Bench: zero-shot visual reasoning tasks (physical, knowledge and symbolic reasoning; 20 subdomains, e.g. mazes, sudoku, science phenomena) posed to image-editing/unified models (input image + instruction) and video generators (reasoning sequence, final frame scored); Gemini-2.5-Pro judges against ground-truth references (validated on a 1,080-output split against three human experts). Result Metric Avg = mean of BC, RO, VQ and RS (%).", "limitations": "Image-editing and unified models without chain-of-thought; the CoT rows (Bagel-Think, GPT-image-1+, Nano Banana+) are other settings and not imported. Single VLM judge." }, "reported_scale": "%", "benchmark_family": "vigor-bench" }, { "id": "vigor-bench-result-avg-video", "name": "ViGoR-Bench zero-shot visual reasoning, video generators (Result Avg)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2603.25823v1#S3.T3", "task": "image-to-video", "setup": { "url": "https://arxiv.org/html/2603.25823v1#S3", "version": "ViGoR-Bench as released with arXiv:2603.25823v1 (26 Mar 2026), Table 3 'Result Metric (%) Avg'", "metric": "Mean of the four Result metrics (BC, RO, VQ, RS), Gemini-2.5-Pro judge with ground-truth references; %, stored /100.", "protocol": "ViGoR-Bench: zero-shot visual reasoning tasks (physical, knowledge and symbolic reasoning; 20 subdomains, e.g. mazes, sudoku, science phenomena) posed to image-editing/unified models (input image + instruction) and video generators (reasoning sequence, final frame scored); Gemini-2.5-Pro judges against ground-truth references (validated on a 1,080-output split against three human experts). Result Metric Avg = mean of BC, RO, VQ and RS (%).", "limitations": "Only the final frame of each generated video is scored by the Result metric; Process metrics (whole sequence) not imported. Single VLM judge." }, "reported_scale": "%", "benchmark_family": "vigor-bench" }, { "id": "phyground-human-overall", "name": "PhyGround physics human evaluation (Overall, TI2V)", "category": "video", "kind": "acc", "chance": 0.2, "url": "https://arxiv.org/html/2605.10806v1#S4.T2", "task": "image-to-video", "setup": { "url": "https://arxiv.org/html/2605.10806v1#S4", "version": "PhyGround as released with arXiv:2605.10806v1 (11 May 2026), Table 2, 250 ti2v prompts", "metric": "Overall human score on a 1-5 scale (equal weight of general adherence and physics-domain views), stored /5 with chance 0.2.", "protocol": "PhyGround: 250 text+image-to-video prompts (first frame given; sources VideoPhy-2, Physics-IQ, OpenVid, WorldModelBench) covering 13 physics laws; each model generates one video per prompt at its native default settings (2,000 videos); 352 quality-controlled human annotators rate 1-5 semantic alignment, physical temporal validity, persistence and the applicable physics laws; Overall = equal-weight combination of the general and physics-domain views.", "limitations": "Only 8 models; native default settings per model (resolution/fps differ); human means of 2.5-3.3 sit in a narrow band. Sub-dimension columns and the PhyJudge automatic scores are not imported." }, "reported_scale": "1-5", "benchmark_family": "phyground" }, { "id": "edival-o-turn3", "name": "EdiVal multi-turn editing overall (EdiVal-O after turn 3)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2509.13399v3#S3.T3", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2509.13399v3#S3", "version": "EdiVal-Agent report v3 (17 Mar 2026), Table 3, 16 editors", "metric": "EdiVal-O after the third sequential edit: geometric mean of EdiVal-IF and EdiVal-CC (0-100), the column the table ranks by; stored /100.", "protocol": "EdiVal-Bench multi-turn editing: each model applies three sequential object-centric edit instructions (T1-T3) per image with default settings (APIs or official repos); EdiVal-IF (instruction following via Qwen2.5-VL-7B plus Grounding DINO checks) and EdiVal-CC (content consistency via DINOv3 features) are combined by geometric mean into EdiVal-O; the table ranks models by EdiVal-O after turn 3.", "limitations": "Automatic object-centric pipeline (detector + small VLM), not human ratings; turn-1/turn-2 values and the perceptual-quality table (Table 4) are not imported." }, "reported_scale": "0-100", "benchmark_family": "edival" }, { "id": "weavebench-avg", "name": "WEAVEBench in-context multi-turn editing (Avg)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2511.11434v1#S4.T2", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2511.11434v1#S3", "version": "WEAVEBench as released with arXiv:2511.11434v1 (14 Nov 2025), Table 2", "metric": "Avg of the Science, Creation, Logic and Game domain scores (GPT-4.1 key-point judge, 0-1).", "protocol": "WEAVEBench: human-annotated multi-turn, in-context image editing and generation tasks (science, creation, logic, game) requiring visual memory and world knowledge; each editor produces the edited image from the original image and instruction history; GPT-4.1 judges key points against the reference image and the original image (hybrid VLM-judge, validated against three experts); Avg = mean over the four task domains. Editors are run in the partial in-context setting with image-only evaluation (row icons).", "limitations": "Single VLM judge; benchmark images partly generated by Seedream 4.0, Nano Banana and SeedEdit 3.0; Seedream 4.0 and Nano Banana rows appear twice under different in-context settings and are not imported." }, "reported_scale": "0-1", "benchmark_family": "weavebench" }, { "id": "knowvis-brief-overall-en", "name": "Knowledge Visualization T2I (Brief Caption, English overall)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2604.22302v1#S4.T2", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2604.22302v1#S3", "version": "Knowledge Visualization benchmark as released with arXiv:2604.22302v1 (24 Apr 2026), Table 2 Brief Caption track", "metric": "Mean checklist accuracy (Qwen2.5-VL-32B binary VQA) averaged over six subjects, English prompts; %, stored /100.", "protocol": "Knowledge Visualization benchmark: 1,800 textbook-grounded science prompts (biology, chemistry, geography, history, math, physics; Chinese and English), Brief Caption track (abstract textbook caption, model must supply the knowledge); each image is checked against 4-6 expert-reviewed checklist items by Qwen2.5-VL-32B-Instruct as binary VQA; score = mean item accuracy; 'Overall en' = average over the six subjects on English prompts.", "limitations": "Single open-weight MLLM judge; Chinese-prompt column and the Detailed Caption track (Table 5) are not imported." }, "reported_scale": "% checklist accuracy", "benchmark_family": "knowledge-visualization" }, { "id": "realpref-bench-human", "name": "RealPref-Bench image editing (human user-study score)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2604.19406v1#S5.T1", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2604.19406v1#S5", "version": "RealPref-Bench as released with arXiv:2604.19406v1 (21 Apr 2026), Table 1 'Human' column", "metric": "Mean human score on the 0-5 rubric (0 = failure, 3 = instruction mostly followed but poor quality, 5 = full compliance with high quality); stored /5.", "protocol": "RealPref-Bench: real-world images with manually verified edit instructions over eight sub-tasks (add, remove, object swap, background replace, colour, bokeh, relighting, style); user study (Sec. 5.4): five annotators score over 1k edited outputs per the 0-5 rubric (instruction adherence and image quality); 'Human' = average annotator score per model.", "limitations": "Five annotators; the benchmark was built by the authors of an editing method (HP-Edit, not imported); HP-score columns (VLM scorer) not imported." }, "reported_scale": "0-5", "benchmark_family": "realpref-bench" }, { "id": "switch-i2v-vlm-overall", "name": "SWITCH interactive-interface I2V (VLM-judge overall)", "category": "video", "kind": "acc", "chance": 0.2, "url": "https://arxiv.org/html/2511.17649v5#S4.T3", "task": "image-to-video", "setup": { "url": "https://arxiv.org/html/2511.17649v5#S4", "version": "SWITCH report v5 (7 Jul 2026), Table 3 single-image-to-video block, VLM-as-Judge Overall", "metric": "Mean Gemini 3.1 Pro overall score on a 1-5 scale per video, stored /5 with chance 0.2.", "protocol": "SWITCH generative track: image-to-video generation of tangible-interface interactions (e.g. operating light switches, appliance panels) from an initial frame plus a prompt fixing task type, next action, stop condition and required visible evidence; Gemini 3.1 Pro judges each video (1,170 valid evaluations) on scene fidelity, control correctness, procedure correctness and execution observability and gives a direct overall score on a 1-5 scale.", "limitations": "Single VLM judge; the 160-video human evaluation in the same table reshuffles the ranking and is not imported; video-to-video block not imported. ShareAlike licence." }, "reported_scale": "1-5", "benchmark_family": "switch" }, { "id": "wrop-object-permanence-human-elo", "name": "WROP object-permanence exam (human Elo)", "category": "video", "kind": "elo", "url": "https://arxiv.org/html/2609.28654v1#S5.T2", "task": "video-editing", "setup": { "url": "https://arxiv.org/html/2609.28654v1#S5", "version": "WROP exam, arXiv:2609.28654v1 (23 Sep 2026), Table 2 human preference leaderboard (14 models)", "metric": "Bradley-Terry Elo from 361 blind pairwise human judgments; 95% CI kept in notes.", "protocol": "WROP exam: 300 object-permanence questions (six task families); each model receives the conditioning video prefix through its API's interface (true continuation, reference-to-video or frame-wise edit/transfer; Table 1) and its predicted continuation is trimmed to the target span; twenty raters made 361 blind pairwise judgments (50-52 per model, ties 0.5); Bradley-Terry maximum-likelihood Elo with one virtual draw per model.", "limitations": "Small sample (about 52 games per model, wide CIs); models are reached through different API interfaces (continuation, reference-to-video, edit/transfer), so the test mixes capability with interface; ratings relative to this 14-model pool." }, "reported_scale": "Elo", "benchmark_family": "wrop" }, { "id": "prost-zero-shot-macro", "name": "PROST physical reasoning (zero-shot, 10-concept macro average)", "kind": "acc", "chance": 0.25, "category": "llm", "url": "https://aclanthology.org/2021.findings-acl.404.pdf", "description": "PROST (Aroca-Ouellette et al., Findings of ACL-IJCNLP 2021): 18,736 four-option multiple-choice questions from 14 templates over 10 physical concepts (direction, mass, height, circumference, stackable, rollable, graspable, breakable, slideable, bounceable), asked zero-shot. Score is the macro average of the 10 concept accuracies (multi-template concepts averaged first). Chance 0.25 as stated in the paper ('random guessing would yield an accuracy of 25%'). T5/UnifiedQA rows skip the sliding concept (asterisked 9-concept average) and are not comparable; UnifiedQA is also fine-tuned." }, { "id": "mt-bench-gpt4-single", "name": "MT-Bench (GPT-4 single-answer grading, LMSYS runs)", "kind": "linear", "latent_scale": 1, "unit": "MT-Bench score (1-10)", "category": "llm", "url": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard/blob/6f2eeeba09bf42144dbc118fe1a8e12e4c533f9d/leaderboard_table_20250804.csv", "metric": "mean GPT-4 single-answer grade (1-10) over the 80 MT-Bench questions, both turns", "description": "MT-Bench (Zheng et al. 2023, arXiv 2306.05685): 80 two-turn open-ended questions in 8 categories (writing, roleplay, reasoning, math, coding, extraction, STEM, humanities); GPT-4 grades each answer 1-10 (single-answer grading, fastchat.llm_judge) and the score is the mean over all 160 turns. Raw judge-score units kept (kind linear, fixed latent_scale 1 = one MT-Bench point), not converted to accuracy. Only rows LMSYS published in its 2023 leaderboard tables (its own fastchat.llm_judge runs) are recorded; later rows whose values equal the makers' own announcements (e.g. Zephyr, Mixtral, Qwen1.5, Mistral 7B Instruct) are left out because LMSYS may have copied them." }, { "id": "mm-mt-bench", "name": "MM-MT-Bench (Mistral, GPT-4o judge)", "kind": "linear", "latent_scale": 1, "unit": "MM-MT-Bench score (1-10)", "category": "llm", "url": "https://arxiv.org/html/2410.07073v2#S3", "metric": "mean judge grade (1-10) over the 92 MM-MT-Bench conversations", "description": "Multimodal MT-Bench (Mistral AI, Pixtral 12B paper Sec. 3; dataset mistralai/MM-MT-Bench, eval split): 92 conversations over charts, tables, PDF pages, diagrams and miscellaneous images (69 single-turn, 23 multi-turn); an independent LLM judge (GPT-4o in the paper's Table 2) grades each response 1-10 against a reference answer for correctness and completeness; score = mean grade. Raw judge units (kind linear, fixed latent_scale 1 = one point), not accuracy." }, { "id": "molmo-human-pref-elo-2409", "name": "Ai2 Molmo human preference Elo (Sept 2024, 27 VLMs)", "kind": "elo", "category": "llm", "url": "https://huggingface.co/allenai/Molmo-72B-0924/blob/ab586dd9640de059566975b8e5bcd0c6fdc73956/README.md#evaluations", "metric": "Elo rating from Ai2's pairwise human preference study", "description": "Human preference evaluation run by Ai2 for the Molmo release (Deitke et al., arXiv 2409.17146): ~15k diverse image+text prompts were sent to 27 vision-language models, and ~870 annotators made ~325k pairwise preference judgments; ratings are Elo/Bradley-Terry scores published in the Molmo model cards ('Human Preference Elo Rating'). One fixed study (Sept 2024); not LMArena vision." }, { "id": "molmo2-human-pref-elo", "name": "Ai2 Molmo2 human preference Elo (video/image QA, 2026)", "kind": "elo", "category": "llm", "url": "https://arxiv.org/html/2601.10611v4#S4.T2", "metric": "Bradley-Terry Elo from Ai2's pairwise human preference study", "description": "Human preference study run by Ai2 for the Molmo2 report (arXiv 2601.10611, Sec. 4.1): 450 manually filtered open-ended questions from annotators plus 51 video-captioning queries; two models' outputs are compared pairwise by annotators (over 105K ratings, 501 per model pair, 21 models); Elo via the Bradley-Terry model. Distinct from the Sept 2024 Molmo study and from LMArena." }, { "id": "molmo2-captest-f1", "name": "Molmo2-CapTest dense video captioning F1", "kind": "acc", "chance": 0, "category": "llm", "url": "https://arxiv.org/html/2601.10611v4#S4.T2", "metric": "statement-level F1 (LLM judge) vs human captions", "description": "Molmo2-CapTest (Ai2, arXiv 2601.10611): 693 Creative Commons videos, each with at least four human-written captions; an LLM judge scores the precision and recall of statements in the model's caption against the annotators' statements, reported as F1 (0-1)." }, { "id": "molmo2-videocount-acc", "name": "Molmo2-VideoCount accuracy (val)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://arxiv.org/html/2601.10611v4#S4.T2", "metric": "exact count accuracy", "description": "Molmo2-VideoCount (Ai2, arXiv 2601.10611): 533 diverse video counting questions (objects, actions, animals; up to 60 instances) built with the Molmo2-VideoPoint pipeline; accuracy = exact-count match on the val split." }, { "id": "humanevalexplain", "name": "HumanEvalExplain (HumanEvalPack, 6-language average)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://arxiv.org/html/2405.04324v1#S6.T10", "description": "HumanEvalPack explanation task (Muennighoff et al. 2023): the model explains a reference solution in natural language, then regenerates the code from its own explanation alone; pass@1 of the regenerated code, averaged over Python, JavaScript, Java, Go, C++ and Rust as printed by the source; greedy. Distinct from HumanEvalSynthesize and HumanEvalFix." }, { "id": "humanevalfix", "name": "HumanEvalFix (HumanEvalPack, 6-language average)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://arxiv.org/html/2405.04324v1#S6.T11", "description": "HumanEvalPack bug-fixing task (Muennighoff et al. 2023): the model repairs a subtly buggy solution given failing unit tests; pass@1 averaged over Python, JavaScript, Java, Go, C++ and Rust as printed by the source; greedy. Distinct from HumanEvalSynthesize and HumanEvalExplain." }, { "id": "paradee-wikitext200-utmos22", "name": "Paradee small-TTS comparison · 200 WikiText-103 sentences · UTMOS (utmos22_strong)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2610.06817v1", "metric": "predicted_mos_utmos", "setup": "Paradee report (arXiv:2610.06817v1) §5 and Table 3: each fixed-voice system reads the same 200 held-out English sentences (held out from a 12,000-sentence WikiText-103 corpus) in its own voice; naturalness predicted with UTMOS (utmos22_strong checkpoint), averaged over the 200 sentences. One author ran all systems.", "scale_transform": "score = (raw UTMOS - 1) / 4", "raw_scale_min": 1, "raw_scale_max": 5, "display_unit": "predicted MOS / 5; not human MOS, not accuracy", "caveats": [ "Automatic MOS predictor; the paper notes UTMOS missed an audible buzz artefact (§7, §9).", "Each system uses a different voice (af_heart, lessac, Bella), so the score mixes voice and model quality." ] }, { "id": "paradee-wikitext200-whisper-base-wer", "name": "Paradee small-TTS comparison · 200 WikiText-103 sentences · Whisper-base WER", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2610.06817v1", "metric": "word_error_rate", "setup": "Same 200 held-out WikiText-103 sentences as paradee-wikitext200-utmos22; audio transcribed with Whisper (base) and compared with the source text (§5). WER % / 100.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Whisper base is a small ASR model; the paper says WER 'barely separates our models'.", "Normalisation not stated." ] }, { "id": "carun-tts-bench56-whisper-v3-wer", "name": "carun tts-bench · 56-sentence English corpus · Whisper large-v3 WER", "category": "tts", "kind": "err", "url": "https://github.com/carun/tts-bench/blob/27727b24abbc688f718992eb5bfc5c7121facfcb/README.md", "metric": "word_error_rate", "setup": "carun/tts-bench (Apache-2.0): 56 English sentences in 7 categories (prose, numbers, acronyms, jargon, names, complex clauses, heteronyms) synthesised once per engine with its default/preset voice; each output transcribed by openai/whisper-large-v3 and scored against the input after the Whisper English normaliser on both sides; overall WER as printed in data/REPORT_DATA.md. The Parakeet-judge column is a correlated second ASR view of the same audio and is not recorded.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Small corpus (56 sentences, one seed); the author notes the heteronym category measures nothing.", "Fixed-voice systems each use their own preset voice." ] }, { "id": "wayu-libritts500-en-whisper-v3-cer", "name": "Wayu Thai TTS eval · English CER on 500 held-out LibriTTS lines (Whisper Large V3)", "category": "tts", "kind": "err", "url": "https://arxiv.org/html/2609.03502v1", "metric": "character_error_rate", "setup": "Wayu/Paxa/Typhoon fixed-voice TTS report (arXiv:2609.03502v1) Table 8: each system synthesises 500 held-out LibriTTS English transcript lines with its Thai fixed/reference voice; audio transcribed with Whisper Large V3; mean CER in percent (divided by 100 here). Same evaluator and run as wayu-thai-cer500-typhoon-whisper.", "score_unit": "fraction (source percent / 100)", "setup_sources": [ "https://arxiv.org/html/2609.03502v1" ], "caveats": [ "Run by the makers of Wayu-Paxa-TTS-Edge, which was trained on LibriTTS transcripts (held-out lines are used for the test).", "English intelligibility of Thai-voiced systems; complements the Thai CER of the same table rather than repeating it." ] }, { "id": "ultratext-bench-composite", "name": "UltraText Bench (Composite, bilingual dense text rendering)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2610.09823v1#S5.T5", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2610.09823v1#S4", "version": "UltraText Bench as released with arXiv:2610.09823v1 (7 Oct 2026), Table 5 leaderboard (24 model configurations)", "metric": "Composite: 60/30/5/5 weighted Text Fidelity / Text Clarity / Spatial / Scene quality (Q-Judger ratings 0-100), prompt-macro mean per language, English and Chinese averaged equally; stored /100.", "protocol": "432 human-reviewed prompts (24 real-world scene categories in 6 domains, 3 difficulty levels L1 Hard / L2 Very Hard / L3 Extreme, 216 English + 216 Chinese), each with 4-12 exact target strings and a structured per-region reference (content, coarse position, carrier, visual attributes); prompt-only generation with each model's standard sampling configuration; Q-Judger VLM (from Qwen-Image-Bench) returns six image-level scores under the paper's rubric; accuracy+completeness form Fidelity, position+layout form Spatial.", "limitations": "Single VLM judge (validated against ten human participants on a subset); model revisions, resolutions and API dates not published; execution failures lower coverage, not the quality mean; benchmark authors overlap with the LLaDA-Image team." }, "reported_scale": "0-100", "benchmark_family": "ultratext-bench" }, { "id": "creval-bench-overall", "name": "CREval-Bench creative image editing (Overall, GPT-4o QA evaluator)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2603.26174v1#S3.T2", "task": "image-editing", "setup": { "url": "https://arxiv.org/html/2603.26174v1#S3", "version": "CREval-Bench as released with arXiv:2603.26174v1 (27 Mar 2026), Table 2", "metric": "Overall avg: weighted average of Instruction Following, Visual Consistency and Visual Quality QA scores (0-100) over three creative editing types; stored /100.", "protocol": "800+ creative image-manipulation samples (Customization, Contextualization, Stylization; nine creative dimensions) with 13K automatically generated evaluation questions and reference answers; GPT-4o answers each question about the edited image and a matching answer earns the score; IF, VC and VQ aggregated by weighted average.", "limitations": "Single MLLM evaluator (GPT-4o); per-model inference settings not tabulated; user study reports agreement with human judgments." }, "reported_scale": "0-100", "benchmark_family": "creval" }, { "id": "servimage-task-acceptance", "name": "ServImage task acceptance (human payment decisions, commercial design tasks)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2604.24023v2#S5.T2", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2604.24023v2#S3", "version": "ServImageBench as released with arXiv:2604.24023v2, Table 2 standard settlement scenario (16 models)", "metric": "Task Acceptance: proportion of tasks whose deliverables are approved and paid under the original contract according to human payment-decision labels; stored as a fraction.", "protocol": "1,070 paid commercial design tasks collected from crowdsourcing platforms (271 portrait, 450 e-commerce product, 349 digital content; text-to-image and image-editing workflows; 2,046 required deliverables, $295k contract value); each model generates the required candidate deliverables from the brief, references and source image; human annotators label each candidate as approved/paid or not under the original contract.", "limitations": "Mixes generation and editing tasks; editing-only models (SeedEdit) attempt all tasks; API tiers/quality settings not stated; ServImageModel-predicted variant (Table 9) not used." }, "reported_scale": "%", "benchmark_family": "servimage" }, { "id": "hpsv3-benchmark", "name": "HPSv3 image generation benchmark (Overall HPSv3 score)", "category": "image", "kind": "linear", "latent_scale": 1, "direction": "higher", "url": "https://github.com/MizzenAI/HPSv3/blob/bd0c5fcb5f587617b0169c07222ab78d01e2f3c2/README.md#-benchmark", "task": "text-to-image", "setup": { "url": "https://github.com/MizzenAI/HPSv3/blob/bd0c5fcb5f587617b0169c07222ab78d01e2f3c2/README.md", "version": "'Image Generation Benchmark of HPSv3' table in the MizzenAI/HPSv3 README @bd0c5fc (11 models)", "metric": "Overall: mean HPSv3 score of each model's generated images across 12 prompt categories (Characters, Arts, Design, Architecture, Animals, Natural Scenery, Transportation, Products, Others, Plants, Food, Science); raw units (about -0.2 to 10.6), higher is better; latent_scale 1 = one HPSv3 point.", "protocol": "Images generated by the HPSv3 authors for their HPDv3 prompt sets and scored with the HPSv3 preference model (Qwen2-VL-7B with an uncertainty-aware ranking head).", "comparability": "Automatic preference-model score, not human votes; HPSv3 was trained on HPDv3, which contains images from several of the benchmarked models. Do not pool with HPS v2 values.", "limitations": "README does not list per-model sampling settings or prompt counts; the HPSv3 paper (arXiv 2508.03789) carries the default arXiv licence, so only the MIT README table is used." }, "reported_scale": "HPSv3 score (raw)", "benchmark_family": "hps" }, { "id": "pai-bench-g-overall", "name": "PAI-Bench-G physical-AI video generation (Overall)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2512.01989v1#S4.T3", "task": "image-to-video", "setup": { "url": "https://arxiv.org/html/2512.01989v1#S4", "version": "PAI-Bench-G as released with arXiv:2512.01989v1 (1 Dec 2025), Table 3 (15 VGMs)", "metric": "Overall = average of Domain Score (MLLM QA accuracy on physical and semantic constraints) and Quality Score (subject/background consistency, motion smoothness, aesthetic and imaging quality, etc.), 0-100; stored /100.", "protocol": "PAI-Bench-G real-world Physical-AI cases in six domains (common sense, autonomous vehicles, robotics, industry, human, physics); each model generates from the case caption and reference image (image-conditioned: the Quality Score includes Image-to-Video subject and background fidelity to the reference image) with default settings; open-source models averaged over five seeds per caption, Veo3 one video per caption.", "limitations": "Automatic metrics plus MLLM judge; Generation settings are each model's defaults; CogVideoX/CogVideoX1.5 (T2V vs I2V checkpoint not stated), LTX-Video-2B (version not stated), Wan2.1-I2V-14B-720P, MAGI-1, DynamiCrafter and Cosmos-Predict rows not mapped." }, "reported_scale": "0-100", "benchmark_family": "pai-bench" }, { "id": "pai-bench-g-human-elo", "name": "PAI-Bench-G human preference arena (Overall ELO)", "category": "video", "kind": "elo", "url": "https://arxiv.org/html/2512.01989v1#A1.T8", "task": "image-to-video", "setup": { "url": "https://arxiv.org/html/2512.01989v1#A1", "version": "PAI-Bench-G human study, arXiv:2512.01989v1 Appendix Table 8 (8 systems incl. source videos)", "metric": "Overall ELO aggregated from pairwise human preferences (video quality and physical plausibility, ties allowed) over videos generated from identical captions.", "protocol": "Arena-based human study on PAI-Bench-G outputs; pairs sampled across models for the same caption.", "limitations": "Number of participants and votes not given in the table; Generation settings are each model's defaults; CogVideoX/CogVideoX1.5 (T2V vs I2V checkpoint not stated), LTX-Video-2B (version not stated), Wan2.1-I2V-14B-720P, MAGI-1, DynamiCrafter and Cosmos-Predict rows not mapped." }, "benchmark_family": "pai-bench" }, { "id": "caparena-auto-score", "name": "CapArena-Auto detailed image captioning (Score_Avg, GPT-4o judge)", "category": "llm", "kind": "linear", "latent_scale": 50, "direction": "higher", "url": "https://arxiv.org/html/2503.12329v1#A10.T8", "setup": { "url": "https://arxiv.org/html/2503.12329v1#S5", "version": "CapArena-Auto leaderboard as printed in arXiv:2503.12329v1 Appendix J Table 8 (22 VLMs)", "metric": "Score_Avg: mean over three baselines of the net pairwise score (+1 win, -1 loss, 0 draw) on 600 images, as printed on a -100..100 scale; raw units, latent_scale 50 (about one logit of win odds per 50 net points near parity).", "protocol": "600 DOCCI test images sampled from 149 clusters; detailed-caption prompt; each caption compared with GPT-4o, CogVLM2-llama3-chat-19B and MiniCPM-V2.6 captions by GPT-4o as judge with the human reference caption as support; 94.3% Spearman correlation with the CapArena human ranking (Table 3).", "limitations": "Single VLM judge (GPT-4o) that is also a baseline and a contestant; caption length correlates with score; generation settings per model not tabulated." }, "reported_scale": "-100..100", "benchmark_family": "caparena" }, { "id": "physbench-test-avg", "name": "PhysBench physical-world understanding (test, Avg)", "category": "llm", "kind": "acc", "chance": 0.25, "url": "https://arxiv.org/html/2501.16411v2#S3.T3", "setup": { "url": "https://arxiv.org/html/2501.16411v2#S3", "version": "PhysBench test set as evaluated in arXiv:2501.16411v2 (Table 3 and Appendix Table 28, 75 VLMs)", "metric": "Average accuracy over the four task categories (Property, Relationships, Scene, Dynamics); four-option multiple choice, chance 25%.", "protocol": "10,002 interleaved video-image-text multiple-choice entries; models evaluated with their own generation settings (Appendix Tables 9-11), with frames merged or sequential per model format.", "limitations": "Mixed input formats (merge vs seq) per model; GPT-4o, Gemini 1.5 and Claude 3.5 Sonnet rows not mapped (snapshot not stated)." }, "reported_scale": "%", "benchmark_family": "physbench" }, { "id": "msearth-mcq-overall", "name": "MSEarth-Bench Earth-science figure MCQ (Overall accuracy)", "category": "llm", "kind": "acc", "chance": 0.25, "url": "https://arxiv.org/html/2505.20740v3#S4.T4", "setup": { "url": "https://arxiv.org/html/2505.20740v3#S4", "version": "MSEarth-Bench MCQ test split as evaluated in arXiv:2505.20740v3 (Table 4)", "metric": "Overall accuracy on 2,784 four-option multiple-choice questions grounded in figures from Earth-science papers (single-image, multi-image and cross-image; reasoning and perception); chance 25%.", "protocol": "Structured prompt per model; choice extracted with regular expressions; chat/instruction-tuned variants.", "limitations": "Domain-specific (geoscience); GPT-4o version truncated in the paper and not mapped; Llama-3.2-90B-Vision path is the base checkpoint and not mapped." }, "reported_scale": "%", "benchmark_family": "msearth" }, { "id": "mmmg-image-avg", "name": "MMMG image generation (average over 15 tasks)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2505.17613v2#A4.T9", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2505.17613v2#S3", "version": "MMMG image-generation subset as evaluated in arXiv:2505.17613v2 (Appendix Table 9, 12 models)", "metric": "Average accuracy over 15 image-generation tasks (object inclusion/exclusion/count/knowledge/commonsense/attribution, comparison and universal relations, relative/absolute spatial, region/border fill, single/double/multilingual text rendering); 0-1.", "protocol": "Synthetic instructions per task; each task scored with the method best aligned with human judgment in the authors' validation (VLM-based or programmatic checks); API/HF checkpoints listed in Appendix C.", "limitations": "Automatic evaluation; Janus Pro and BLIP-3o sizes not stated (not mapped)." }, "reported_scale": "0-1", "benchmark_family": "mmmg" }, { "id": "mmmg-kg-score-avg", "name": "MMMG knowledge-image generation (MMMG-Score, average over education levels)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2506.10963v2#S4.T2", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2506.10963v2#S3", "version": "MMMG benchmark (4,456 prompts) as evaluated in arXiv:2506.10963v2, Table 2", "metric": "MMMG-Score = knowledge fidelity (1 - GED between the reference knowledge graph and the subgraph that OpenAI-o3, high effort, grounds in the generated image) x a readability score from image segment counts, averaged over six education levels; x100 in the paper, stored /100.", "protocol": "Knowledge-intensive explanatory-image prompts across 10 disciplines and six education levels; OpenAI-o3 grounds the expert-verified reference graph in each generated image.", "limitations": "Automatic o3-based grounding (missing items count as false); distinct from the multitask MMMG benchmark (arXiv 2505.17613)." }, "reported_scale": "x100", "benchmark_family": "mmmg-kg" }, { "id": "real-realism-avg", "name": "REAL realism benchmark for T2I (average of attribute, relationship and style scores)", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2502.10663v1#S4.T5", "task": "text-to-image", "setup": { "url": "https://arxiv.org/html/2502.10663v1#S3", "version": "REAL benchmark as reported in arXiv:2502.10663v1, Table 5 (four T2I models)", "metric": "Average of three realism scores in 0-1: attribute (share of visible schema attributes judged correctly depicted, normalised by visible attributes), relationship (objects and unusual relations present and realistic) and style (photorealistic rather than stylised), all from VQA queries.", "protocol": "Prompts from 200 iNaturalist classes and 200 CUB-Birds classes (attributes, style) and UnRel unusual relationships; each image scored by a VQA model with REAL's schema; validated against MTurk annotations on 300 images.", "limitations": "Only four models; SD 3.5 Turbo row not mapped (Large vs Medium not stated)." }, "reported_scale": "0-1", "benchmark_family": "real" }, { "id": "alpacaeval1-winrate", "name": "AlpacaEval 1.0 win rate (vs text-davinci-003, GPT-4 judge)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/tatsu-lab/alpaca_eval/blob/cd543a149df89434d8a54582c0151c0b945c3d20/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv", "metric": "win rate against text_davinci_003 on the 805 AlpacaEval instructions, judged by alpaca_eval_gpt4", "description": "AlpacaEval 1.0: the share of 805 instructions where a GPT-4 judge prefers the model's answer to text-davinci-003's. Not length-controlled; the reference scores 50% by definition." }, { "id": "vectara-hhem-factual-consistency", "name": "Vectara hallucination leaderboard (HHEM-2.3, factual consistency rate)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/vectara/hallucination-leaderboard/blob/af0e3207741054b44e7c87688a40a815ecf446df/README.md", "metric": "share of answered summaries judged factually consistent with the source document by HHEM-2.3", "description": "Each model summarises the same set of documents with a fixed prompt; Vectara's HHEM-2.3 classifier judges whether each summary is supported by its document. Factual consistency rate = 1 − hallucination rate, over the documents the model answered (answer rate reported separately)." }, { "id": "vectara-hhem-factual-consistency-2025", "name": "Vectara hallucination leaderboard, 2025 dataset (HHEM-2.3, factual consistency rate)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/vectara/hallucination-leaderboard/blob/9ceb7826f56f6cdafbbf92c85b3980a7718e66f0/README.md", "metric": "share of answered summaries judged factually consistent with the source document by HHEM-2.3", "description": "The leaderboard's previous document set (frozen October 2025): each model summarises the same documents with a fixed prompt and HHEM-2.3 judges whether each summary is supported. A different document set from the current leaderboard, so it is a separate test." }, { "id": "tau3-airline", "name": "τ³-bench airline (tau2-bench ≥ v1.0.1)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/sierra-research/tau2-bench/tree/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699", "description": "Airline domain of τ³-bench (tau2-bench v1.0.1+, with the 75+ task fixes): pass^1. Not comparable with the original τ²-bench airline domain." }, { "id": "tau3-retail", "name": "τ³-bench retail (tau2-bench ≥ v1.0.1)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/sierra-research/tau2-bench/tree/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699", "description": "Retail domain of τ³-bench (tau2-bench v1.0.1+, with the task fixes): pass^1. Not comparable with the original τ²-bench retail domain." }, { "id": "tau3-telecom", "name": "τ³-bench telecom (tau2-bench ≥ v1.0.1)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/sierra-research/tau2-bench/tree/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699", "description": "Telecom domain of τ³-bench (tau2-bench v1.0.1+): pass^1." }, { "id": "tau3-banking-v101", "name": "τ³-bench banking_knowledge (v1.0.1 grading)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/sierra-research/tau2-bench/tree/4ce7c0397c1eb65c9bbe59aeacfe1ca44a1cd699", "description": "Knowledge-retrieval banking domain of τ³-bench, graded with tau2-bench v1.0.1+ (the release notes say earlier banking scores are not comparable): pass^1." }, { "id": "charxiv-dq", "name": "CharXiv descriptive questions", "kind": "acc", "chance": 0, "category": "llm", "url": "https://arxiv.org/abs/2406.18521", "description": "CharXiv descriptive-question accuracy (validation set): reading basic elements of real scientific charts (titles, axes, labels, counts), graded by GPT-4o. The reasoning questions are a separate benchmark (charxiv-rq)." }, { "id": "xtc-bench-gen-overall", "name": "XTC-Bench generation (Overall Gen., scene-graph node agreement)", "category": "image", "kind": "acc", "url": "https://arxiv.org/html/2604.25072v1#S4.T2", "setup": { "url": "https://arxiv.org/html/2604.25072v1", "version": "arXiv:2604.25072v1", "metric": "Overall Gen.: how well the image generated from a scene-graph-derived prompt recovers the scene-graph nodes (objects, attributes, relations) over all nodes, continuous 0-1.", "protocol": "XTC-Bench derives generation prompts and understanding queries from the same structured scene graphs (COCO-based); generated images are parsed into scene graphs and matched to the reference with an LLM judge; Table 2 reports node-level generation quality. Eight open-source unified multimodal models run by the authors.", "limitations": "Single LLM-judge pipeline; model checkpoints/resolutions not detailed in the main text; CCTA and understanding columns not recorded." }, "reported_scale": "0-1", "chance": 0, "benchmark_family": "xtc-bench" }, { "id": "llm-blender-mixinstruct-gpt-rank", "name": "MixInstruct GPT-Rank (LLM-Blender, mean rank among 11 instruction-tuned LLMs, ChatGPT pairwise judge)", "category": "llm", "kind": "linear", "direction": "lower", "latent_scale": 1, "unit": "mean rank (1-11)", "url": "https://arxiv.org/html/2306.02561v3#S5.T2", "setup": { "url": "https://arxiv.org/html/2306.02561v3", "version": "MixInstruct test split, 4,771 examples (arXiv 2306.02561v3, Table 2)", "metric": "GPT-Rank: each example's 11 candidate answers are ranked via ChatGPT pairwise comparisons (MaxWins aggregation); a model's score is its mean rank over examples (1 = best, 11 = worst).", "protocol": "Eleven open instruction-following LLMs answer 100K MixInstruct prompts (Alpaca-GPT4, Dolly-15K, GPT4All-LAION, ShareGPT); ChatGPT compares all candidate pairs on the 4,771 test examples; checkpoints listed on the dataset card.", "limitations": "Rank is pool-relative (11 models, mostly 2023 open models) and judged by ChatGPT; scale 1 = one rank position." }, "reported_scale": "rank" }, { "id": "vmmu-mean", "name": "VMMU (Vietnamese multimodal understanding and reasoning, mean accuracy over 7 tasks)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2508.13680v5#S4.T2", "setup": { "url": "https://arxiv.org/html/2508.13680v5", "version": "arXiv:2508.13680v5", "metric": "Mean accuracy over the seven task columns of Table 2 (Math, Physics, Chemistry, Biology, Geography, Driving-license, IQ), stored as a fraction.", "protocol": "2.5k Vietnamese multimodal questions in 7 tasks; each question requires combining the image and text (no OCR/text-only shortcuts); models prompted with the paper's fixed prompt (Figs. 9-10); the paper prints a random-answer baseline row (mean 25.92%).", "limitations": "Single prompt, API/open-weight inference settings not detailed; random baseline is 25.92% over the seven tasks (stored chance 0.2592)." }, "reported_scale": "0-1", "chance": 0.2592, "benchmark_family": "vmmu" }, { "id": "medmarks-v-mean-win-rate", "name": "Medmarks-V mean win rate (19 verifiable medical benchmarks, 71 configurations)", "category": "llm", "kind": "linear", "latent_scale": 0.08, "url": "https://arxiv.org/html/2605.01417v1#A2.T4", "setup": { "url": "https://arxiv.org/html/2605.01417v1", "version": "Medmarks-V as run in arXiv 2605.01417v1 (61 models, 71 configurations)", "metric": "Weighted mean win rate: for each of the 19 verifiable benchmarks a model's win rate is its share of pairwise wins against the other configurations; benchmarks are combined with log-size weights (appendix H).", "protocol": "Open-source evaluation suite built on the verifiers library; open-weight models run on vLLM, API models through a gateway; each model evaluated with its officially recommended sampling parameters.", "limitations": "Pool-relative (win rate against the 71-configuration pool), so scores shift if the pool changes; latent_scale 0.08 is the standard deviation (0.081) of the 71 published win rates (mean 0.50)." }, "reported_scale": "win rate (0-1)" }, { "id": "cosmos-3d-pose-success", "name": "Cosmos 3D consistency: camera-pose estimation success rate on generated videos", "category": "video", "kind": "acc", "url": "https://arxiv.org/html/2501.03575v3#S5.T19", "setup": { "url": "https://arxiv.org/html/2501.03575v3", "version": "arXiv:2501.03575v3", "metric": "Share of generated videos on which a structure-from-motion/pose estimator succeeds (higher = more geometrically consistent), stored as a fraction.", "protocol": "NVIDIA's evaluation of base Cosmos models on 3D consistency (Sec. 5.2): videos generated from DAVIS-style prompts are reconstructed with a pose estimator; success rate and epipolar (Sampson) error are reported alongside view-synthesis metrics.", "limitations": "Maker-run, four NVIDIA models only (plus VideoLDM and real-video reference rows not recorded); estimator and prompt set as in the paper." }, "reported_scale": "fraction", "chance": 0, "benchmark_family": "cosmos-3d-consistency" }, { "id": "cosmos-3d-sampson-error", "name": "Cosmos 3D consistency: Sampson epipolar error on generated videos (lower is better)", "category": "video", "kind": "linear", "url": "https://arxiv.org/html/2501.03575v3#S5.T19", "setup": { "url": "https://arxiv.org/html/2501.03575v3", "version": "arXiv:2501.03575v3", "metric": "Mean Sampson (epipolar) error of correspondences between frames of the generated video; lower means more geometrically consistent.", "protocol": "As cosmos-3d-pose-success: same videos, same table (Table 19).", "limitations": "Raw error units; linear scale 0.1; four NVIDIA models only." }, "reported_scale": "Sampson error", "benchmark_family": "cosmos-3d-consistency", "direction": "lower", "latent_scale": 0.1 }, { "id": "kaleidoscope-overall", "name": "Kaleidoscope (in-language multimodal exams, 18 languages, overall accuracy)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2504.07072v2#S3.T2", "setup": { "url": "https://arxiv.org/html/2504.07072v2", "version": "arXiv:2504.07072v2", "metric": "Overall accuracy (Acc.) over all Kaleidoscope questions (multimodal and text-only), format errors counted as incorrect; stored as a fraction.", "protocol": "20,911 multiple-choice exam questions in 18 languages (55% need the image); zero-shot, answer extracted from tags; open-weight models run locally with temperature 0.7, max 1024 tokens, 512x512 images; closed models use chain-of-thought prompts.", "limitations": "Single prompt format; option counts vary by question so random-guess chance is not a single number (chance stored as 0)." }, "reported_scale": "0-1", "chance": 0, "benchmark_family": "kaleidoscope" }, { "id": "factual-recall-premium2k-em-0shot", "name": "Factual knowledge recall, Premium2k exact match (zero-shot)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2404.16164v1#S4.T1", "setup": { "url": "https://arxiv.org/html/2404.16164v1", "version": "arXiv:2404.16164v1", "metric": "Exact-match accuracy (zero-shot) on the 2,000-question Premium2k subset (fact-recall questions generated from Wikidata triplets with Wikipedia-grounded answers), stored as a fraction.", "protocol": "15k evaluation questions across 20 relation types, Premium2k = a 2k subset with manually verified, specific questions; zero-shot or 10-shot prompts per the paper's Tables 5-7; EM against the answer or its Wikidata aliases.", "limitations": "Questions generated with Claude v1.3; sampling and API snapshots (July 2023) as in the paper; EM only (the 'Contains' column is not recorded)." }, "reported_scale": "0-1", "chance": 0, "benchmark_family": "factual-recall-premium2k" }, { "id": "factual-recall-premium2k-em-10shot", "name": "Factual knowledge recall, Premium2k exact match (10-shot)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2404.16164v1#S4.T1", "setup": { "url": "https://arxiv.org/html/2404.16164v1", "version": "arXiv:2404.16164v1", "metric": "Exact-match accuracy (10-shot) on the 2,000-question Premium2k subset (fact-recall questions generated from Wikidata triplets with Wikipedia-grounded answers), stored as a fraction.", "protocol": "15k evaluation questions across 20 relation types, Premium2k = a 2k subset with manually verified, specific questions; zero-shot or 10-shot prompts per the paper's Tables 5-7; EM against the answer or its Wikidata aliases.", "limitations": "Questions generated with Claude v1.3; sampling and API snapshots (July 2023) as in the paper; EM only (the 'Contains' column is not recorded)." }, "reported_scale": "0-1", "chance": 0, "benchmark_family": "factual-recall-premium2k" }, { "id": "cvtg-500l-pned-cosmos3", "name": "CVTG-500L PNED (English long-prompt text rendering, Cosmos 3 report)", "category": "image", "kind": "acc", "url": "https://arxiv.org/html/2606.02800v4#S6.T11", "setup": { "url": "https://arxiv.org/html/2606.02800v4", "version": "arXiv:2606.02800v4", "metric": "PNED: OCR-extracted text matched to target strings per region by Hungarian matching on normalised edit distance, English long-prompt set of 500 prompts; stored as a fraction.", "protocol": "500 prompts sampled from CVTG-2K (Du et al. 2025), each upsampled into a long dense description; OCR then PNED/GNED scoring (Sec. 6.2).", "limitations": "Maker-run (NVIDIA) comparison of Cosmos 3 against open and closed models in one table; custom prompt sets (CVTG-500L, CVTG-102ch) are specific to the Cosmos 3 report." }, "reported_scale": "fraction", "chance": 0, "benchmark_family": "cosmos3-t2i" }, { "id": "cvtg-102ch-pned-cosmos3", "name": "CVTG-102ch PNED (Chinese character rendering, Cosmos 3 report)", "category": "image", "kind": "acc", "url": "https://arxiv.org/html/2606.02800v4#S6.T11", "setup": { "url": "https://arxiv.org/html/2606.02800v4", "version": "arXiv:2606.02800v4", "metric": "PNED on a 102-prompt Chinese-character set; stored as a fraction.", "protocol": "102 NVIDIA-written prompts covering public spaces, outdoor scenes and digital displays; OCR + PNED as for CVTG-500L.", "limitations": "Maker-run (NVIDIA) comparison of Cosmos 3 against open and closed models in one table; custom prompt sets (CVTG-500L, CVTG-102ch) are specific to the Cosmos 3 report." }, "reported_scale": "fraction", "chance": 0, "benchmark_family": "cosmos3-t2i" }, { "id": "hpsv3-cosmos3", "name": "HPSv3 score of Cosmos 3 report text-to-image evaluation images", "category": "image", "kind": "linear", "url": "https://arxiv.org/html/2606.02800v4#S6.T11", "setup": { "url": "https://arxiv.org/html/2606.02800v4", "version": "arXiv:2606.02800v4", "metric": "Mean HPSv3 human-preference score of the images generated for the report's T2I benchmark prompts (raw HPSv3 units).", "protocol": "Images generated by each model for the report's T2I prompts and scored by HPSv3.", "limitations": "Maker-run (NVIDIA) comparison of Cosmos 3 against open and closed models in one table; custom prompt sets (CVTG-500L, CVTG-102ch) are specific to the Cosmos 3 report." }, "reported_scale": "HPSv3 score", "benchmark_family": "cosmos3-t2i", "latent_scale": 1 }, { "id": "codereviewqa-acr-exact-match", "name": "CodeReviewQA: automated code refinement, exact match rate", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2503.16167v2#S6.T8", "setup": { "url": "https://arxiv.org/html/2503.16167v2", "version": "arXiv:2503.16167v2", "metric": "Overall exact-match rate (%) of the model's refined code against the human-refined reference over nine programming languages; stored as a fraction.", "protocol": "CodeReviewQA (9 languages, review comments with code); zero-shot greedy generation, exact match after normalisation.", "limitations": "Greedy single run; models listed in the paper's Table 7 only." }, "reported_scale": "0-1", "chance": 0, "benchmark_family": "codereviewqa" }, { "id": "codereviewqa-ctr-invariant-accuracy", "name": "CodeReviewQA: change type recognition, invariant accuracy", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2503.16167v2#A6", "setup": { "url": "https://arxiv.org/html/2503.16167v2", "version": "arXiv:2503.16167v2", "metric": "Invariant accuracy (%) on the change-type-recognition multiple-choice probe: a question counts only if the model picks the right answer under every ordering of the options; stored as a fraction.", "protocol": "MCQA probes derived from the same code-review items as the refinement task; log-probability ranking of the option symbols for every option ordering; zero-shot.", "limitations": "Invariant accuracy is stricter than plain accuracy (chance is below 1/N, stored 0)." }, "reported_scale": "0-1", "chance": 0, "benchmark_family": "codereviewqa" }, { "id": "codereviewqa-clh-invariant-accuracy", "name": "CodeReviewQA: change localisation (hard), invariant accuracy", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2503.16167v2#A6", "setup": { "url": "https://arxiv.org/html/2503.16167v2", "version": "arXiv:2503.16167v2", "metric": "Invariant accuracy (%) on the hard change-localisation multiple-choice probe, order-invariant as above; stored as a fraction.", "protocol": "MCQA probes derived from the same code-review items as the refinement task; log-probability ranking of the option symbols for every option ordering; zero-shot.", "limitations": "Invariant accuracy is stricter than plain accuracy (chance is below 1/N, stored 0)." }, "reported_scale": "0-1", "chance": 0, "benchmark_family": "codereviewqa" }, { "id": "codereviewqa-sih-invariant-accuracy", "name": "CodeReviewQA: solution identification (hard), invariant accuracy", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2503.16167v2#A6", "setup": { "url": "https://arxiv.org/html/2503.16167v2", "version": "arXiv:2503.16167v2", "metric": "Invariant accuracy (%) on the hard solution-identification multiple-choice probe, order-invariant as above; stored as a fraction.", "protocol": "MCQA probes derived from the same code-review items as the refinement task; log-probability ranking of the option symbols for every option ordering; zero-shot.", "limitations": "Invariant accuracy is stricter than plain accuracy (chance is below 1/N, stored 0)." }, "reported_scale": "0-1", "chance": 0, "benchmark_family": "codereviewqa" }, { "id": "think-bench-accuracy", "name": "THINK-Bench final-answer accuracy (reasoning models)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2505.22113v1#S4.T2", "setup": { "url": "https://arxiv.org/html/2505.22113v1", "version": "arXiv:2505.22113v1", "metric": "Accuracy column of Table 2: share of questions answered correctly (judge-checked) by the reasoning model, stored as a fraction.", "protocol": "THINK-Bench: multidisciplinary reasoning questions with annotated key reasoning steps; eleven reasoning models queried once per question; Claude 3.7 Sonnet as step-matching and answer judge.", "limitations": "Empty or answer-less outputs (common for DeepSeek distilled models) count as wrong; single judge model." }, "reported_scale": "0-1", "chance": 0, "benchmark_family": "think-bench" }, { "id": "cedit-bench-en-overall", "name": "CEdit-Bench-EN overall (G_O, GPT-4o judge)", "category": "image", "kind": "acc", "url": "https://arxiv.org/html/2512.07584v1", "description": "CEdit-Bench (introduced in the LongCat-Image report: 1,464 bilingual editing pairs across 15 task categories), English instructions, overall score G_O on the 0–10 VLM-judge scale (GPT-4o), divided by 10. Not GEdit-Bench." }, { "id": "cedit-bench-cn-overall", "name": "CEdit-Bench-CN overall (G_O, GPT-4o judge)", "category": "image", "kind": "acc", "url": "https://arxiv.org/html/2512.07584v1", "description": "CEdit-Bench Chinese-instruction track, overall score G_O on the 0–10 VLM-judge scale (GPT-4o), divided by 10. Separate from the English track." }, { "id": "omnicontext-average", "name": "OmniContext average (GPT-4.1 judge)", "category": "image", "kind": "acc", "url": "https://arxiv.org/html/2506.18871v4", "description": "OmniContext (introduced in the OmniGen2 report): in-context, subject-driven generation from reference images over eight subtasks (single, multiple and scene; character, object and both). Average of the subtask overall scores (geometric mean of prompt following and subject consistency, GPT-4.1 judge, 0–10), divided by 10." }, { "id": "seed-tts-zh-eres2net-sim", "name": "Seed-TTS test-zh · speaker similarity (ERes2Net SV)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2607.23938v1", "description": "Zero-shot voice cloning on Seed-TTS-Eval test-zh; cosine similarity of ERes2Net speaker-verification embeddings between generated and reference speech. Kept apart from the WavLM-based seed-tts-zh-wavlm-sim." }, { "id": "seed-tts-en-eres2net-sim", "name": "Seed-TTS test-en · speaker similarity (ERes2Net SV)", "category": "tts", "kind": "acc", "url": "https://arxiv.org/html/2607.23938v1", "description": "Zero-shot voice cloning on Seed-TTS-Eval test-en; cosine similarity of ERes2Net speaker-verification embeddings between generated and reference speech. Kept apart from the WavLM-based seed-tts-en-wavlm-sim." }, { "id": "wmt26-gemba-esa", "name": "WMT26 General MT (GEMBA-ESA, LLM judge)", "category": "llm", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2609.13916v1", "description": "WMT 2026 General MT test set (document-level, four domains, ~50 languages), scored reference-free by an LLM judge with the GEMBA-ESA adequacy + fluency template (mean of the two), segment-weighted per document; 0–100 divided by 100. Judge model per note." }, { "id": "wmt25-terminology", "name": "WMT25 Terminology Translation (term success rate)", "category": "llm", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2609.13916v1", "description": "WMT25 Terminology Translation Task, proper-terminology condition: share of glossary pairs whose prescribed target term appears in the translation when the source term occurs (corpus-level), micro-averaged over the language pairs and tracks in the note." }, { "id": "vabench-t2av-visual-qa", "name": "VABench T2AV micro Visual QA", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2512.09299v2#S3.T1", "task": "text-to-video", "reported_scale": "0-1", "benchmark_family": "vabench", "setup": { "url": "https://arxiv.org/html/2512.09299v2#S3", "version": "VABench as released with arXiv:2512.09299v2 (6 Apr 2026), text-to-audio-video (T2AV) track, 778 samples", "metric": "Micro-level Visual QA: for each sample, 3-7 human-verified questions about visual elements and physical action details in the prompt; an LLM judge marks each as satisfied or not; score = mean over samples of satisfied/asked (Eq. 2), 0-1.", "protocol": "Each model generates one video per T2AV prompt with default frame rate/duration at 720P (VABench paper) or the evaluator's stated resolution; the VABench toolkit scores the output.", "limitations": "Only the Visual QA column is used (one metric; the other 12 VABench metrics are correlated sub-scores of the same outputs and are not imported). Judge model not named in the extracted text. V+A pipelines (video model + separate audio model) are mapped only through the merged visual cells that belong to the video generator. I2AV track (Table 2) not used." }, "why_new": "No VABench id exists. 8 catalogue models across two CC BY sources (VABench paper: sora-2, veo-3-fast, wan-2-5-preview, seedance-1-0-lite, wan-2-2-5b; Kandinsky 6.0 report: ltx-2-5, kandinsky-6-0-pro, kandinsky-6-0-lite).", "part": "A" }, { "id": "cosmos-hue-t2v-human", "name": "Cosmos HUE human evaluation, text-to-video (Cosmos 3 report)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2606.02800v4#S6.T14", "task": "text-to-video", "reported_scale": "0-100 (% Yes)", "benchmark_family": "cosmos-hue", "setup": { "url": "https://arxiv.org/html/2606.02800v4#S6", "version": "Cosmos 3 report v4 (23 Jun 2026), Table 14 'Cosmos HUE Text-to-Video' (per-dimension breakdown in Appendix Table 32)", "metric": "Per-video HUE = share of atomic questions answered Yes by human annotators (Eq. 12), averaged; %, stored /100. Real-video ground truth scores 93.6.", "protocol": "Human annotators verify single-fact Yes/No/Unclear questions on four dimensions (semantic alignment, physical laws, geometric reasoning, visual integrity) for videos generated from the PAIBench-G prompt set (same generation settings as Table 12).", "limitations": "NVIDIA-run human protocol; Cosmos rows lab-reported; Wan 2.6 not rated. Correlated with pai-bench-g-t2v-cosmos3 (same videos, different judge)." }, "why_new": "Human-verified protocol; 8 catalogue models in one run (6 independent).", "part": "A" }, { "id": "cosmos-hue-i2v-human", "name": "Cosmos HUE human evaluation, image-to-video (Cosmos 3 report)", "category": "video", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2606.02800v4#S6.T14", "task": "image-to-video", "reported_scale": "0-100 (% Yes)", "benchmark_family": "cosmos-hue", "setup": { "url": "https://arxiv.org/html/2606.02800v4#S6", "version": "Cosmos 3 report v4 (23 Jun 2026), Table 14 'Cosmos HUE Image-to-Video' (per-dimension breakdown in Appendix Table 33)", "metric": "Per-video HUE = share of atomic questions answered Yes by human annotators (Eq. 12), averaged; %, stored /100. Real-video ground truth scores 94.4.", "protocol": "As cosmos-hue-t2v-human with image conditioning (PAIBench-G I2V prompt set).", "limitations": "NVIDIA-run human protocol; Cosmos rows lab-reported; Wan 2.6 not rated; Wan2.1-I2V-14B row not mapped. Correlated with pai-bench-g-i2v-cosmos3." }, "why_new": "Human-verified protocol; 7 catalogue models in one run (5 independent).", "part": "A" }, { "id": "math-l5-zeroeval", "name": "MATH Level 5 (ZeroEval, 721 problems)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/WildEval/ZeroEval/blob/8c1485edf12c6efb5f69135a562927c5ad484059/result_dirs/math-l5.summary.json", "description": "ZeroEval's MATH Level 5 task: the 721-problem AI-MO/aimo-validation-math-level-5 set (MATH test Level 5 problems with integer answers), zero-shot with reasoning and a JSON-formatted answer, greedy decoding, field Acc. Not Epoch's math-level-5 (all MATH Level 5 test problems, different harness) and not full MATH or MATH-500.", "why_new": "Different problem subset and harness from math-level-5 (Epoch). 35 catalogue models in the single ZeroEval run (one evaluator, one setup), so the run counts on its own." }, { "id": "livebench-20240726", "name": "LiveBench 2024-07-26", "category": "llm", "kind": "acc", "url": "https://github.com/LiveBench/LiveBench/blob/24364d65076429adfcba7be18af4d44fddc43dce/changelog.md", "description": "LiveBench release 2024-07-26 overall average (0–1): the 1,000 public questions live at that release (math 232, reasoning 150, coding 128, language 140, data analysis 150, instruction following 200), objective ground-truth scoring, mean of the six category means where each category is the mean of its task means (LiveBench show_livebench_result.py). Not merged with livebench-20241125 (different question set)." }, { "id": "vhelm-mmmu-val-em", "name": "MMMU val (VHELM, zero-shot exact match)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2410.07112v2", "description": "MMMU validation split as run by Stanford CRFM's VHELM (HELM MMMUScenario: validation split, per subject × question type, multiple-choice and open), zero-shot prompting, exact-match accuracy averaged over HELM's MMMU sub-runs as printed in VHELM Table 4. Kept apart from 'mmmu' because HELM macro-averages its per-subject runs rather than reporting micro accuracy over the 900 questions." }, { "id": "vhelm-mathvista-testmini-em", "name": "MathVista testmini (VHELM, zero-shot exact match)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2410.07112v2", "description": "MathVista testmini as run by VHELM (HELM MathVistaScenario: testmini split filtered into grade × question-type sub-scenarios for elementary school, high school, college and daily life), zero-shot, exact match, averaged over the sub-runs as printed in VHELM Table 4. Not merged with mathvista-mini (different aggregation and grade filter)." }, { "id": "danish-asr-5set-mean-wer", "name": "Danish ASR Leaderboard · mean WER over 5 Danish test sets (RyeAI)", "category": "stt", "kind": "err", "url": "https://github.com/Rye-A1/danish-asr-leaderboard/blob/f4a0ddc2d6703c43130788507a89231a44f32d1d/README.md", "description": "Macro-average of corpus-level WER (fraction) over the five core Danish test sets of the RyeAI Danish Speech-to-Text Leaderboard: CoRal conversation, CoRal read-aloud, FT Speech, Common Voice 17 (da) and FLEURS (da), with the leaderboard's Danish text normalisation (NFKC, numeral handling, lowercasing, punctuation and filler removal) applied to references and hypotheses. Lower is better; may exceed 1 with insertions." }, { "id": "dutch-asr-3set-avg-wer", "name": "Dutch ASR Leaderboard · average WER (FLEURS, VoxPopuli, MLS nl)", "category": "stt", "kind": "err", "url": "https://huggingface.co/spaces/tvosch/Dutch-ASR-Leaderboard/blob/939431c9206d1e7676347a95d13317f9a3226181/README.md", "description": "Plain mean of WER (fraction) on the Dutch test sets FLEURS nl (364 utterances), VoxPopuli nl (1,137) and MLS nl (3,075), normalised transcripts, as computed by the tvosch/Dutch-ASR-Leaderboard Space. Lower is better." }, { "id": "cose-e-eu4-mean-wer", "name": "CoSE-E enterprise code-switching · 4 European pairs mean WER", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2609.35645v1", "metric": "word_error_rate", "setup": "CoSE-E (arXiv:2609.35645v1): 1,212 linguistically validated enterprise-domain code-switched utterances (customer service / IT support) in five language pairs, synthesized with ElevenLabs Multilingual v2 (European pairs) and v3 (ZH/EN), 10 voices, 24 kHz. Eight ASR systems run by the authors with native automatic language detection and no language ID (Table 8). WER = mean of per-utterance WER per pair. Score = unweighted mean of the four European pairs' WER (ES/EN, FR/EN, FR-CA/EN, DE/EN; Friedman N = 918 European utterances), derived from Table 2 cells.", "score_unit": "error ratio (derived mean of four printed per-pair WERs)", "caveats": [ "Synthetic TTS audio (ElevenLabs); the paper notes possible vendor bias in favour of ElevenLabs Scribe v2 and that results are an upper bound (no telephony, disfluency or noise).", "Macro mean over four pairs is derived, not printed; ZH/EN (Chinese-script WER/CER) excluded because three systems have no ZH/EN result.", "Gemini-3-Flash (preview) and Qwen3-Omni-Instruct rows not imported: they are general LLM/omni models, not stt catalogue entries.", "Whisper-large-v3-turbo row kept but invalid: under automatic language detection it translates instead of transcribing (paper reports it separately)." ] }, { "id": "iberian-asr-7lang-macro-wer", "name": "Iberian ASR benchmark · 7-language macro WER (eu, ca, gl, pt, es + de, tr)", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2609.36920v1", "metric": "word_error_rate", "setup": "arXiv:2609.36920v1 (Telefónica / UAM): 85 hours of read speech, broadcast media and audiobooks from openly licensed corpora (Table 1: e.g. OpenSLR 69/76/77, Common Voice-derived sets) in Basque, Catalan, Galician, Portuguese and Spanish with German and Turkish as controls; eleven systems run by the authors with default settings on one RTX 3090. WER per language, then unweighted macro-average across the seven languages (Eq. 1), as printed in Table 3.", "score_unit": "error ratio (printed % / 100)", "caveats": [ "Languages outside a model's training set (e.g. Basque/Catalan/Galician/Turkish for Parakeet v3 and Canary v2) dominate their macro WER; the score measures coverage as much as accuracy.", "Datasets differ in domain per language (paper §3.3), so per-language comparisons are cautioned." ] }, { "id": "mu-bench-batch-uer", "name": "mu-bench voice-agent caller utterances · batch UER (5 locales)", "category": "stt", "kind": "err", "url": "https://arxiv.org/html/2609.32082v1", "metric": "utterance_error_rate_llm_judge", "setup": "mu-bench (arXiv:2609.32082v1): 4,270 caller utterances from 250 phone calls to an AI banking agent in en-US, es-MX, tr-TR, vi-VN and zh-CN, centred on form-field inputs (names, emails, confirmation codes); 8 kHz telephony audio sent to each provider's batch endpoint with the locale as language hint. Utterance Error Rate = share of utterances an LLM judge (gpt-6-astra) rules a significant (meaning-changing) error after reference-guided LLM normalization; macro-averaged over locales ('all' column of Table 5). Judge agreement with native-speaker raters kappa 0.78.", "score_unit": "error ratio (share of utterances with a significant error)", "caveats": [ "LLM-judged semantic metric, not WER.", "Streaming rows and Azure Speech (no catalogue identity) not imported; xAI 'Grok STT' row not imported (version not stated).", "Providers identified by product name only (no API model strings printed)." ] }, { "id": "cv3-eval-zh-cer", "name": "CV3-Eval ZH (zero-shot Chinese) · CER", "category": "tts", "kind": "err", "url": "https://github.com/FunAudioLLM/CV3-Eval", "metric": "character_error_rate", "setup": "CV3-Eval (CosyVoice 3 release) zero-shot voice-cloning Chinese subset (Hugging Face yuekai/CV3-Eval split zero_shot_zh), official protocol: Seaco Paraformer large CER. Sibling of cv3-eval-en-wer (zero-shot English) and cv3-eval-hard-zh-cer (hard Chinese texts); different text set from the hard split.", "score_unit": "error ratio (source % / 100)", "caveats": [ "Evaluator re-runs released checkpoints with default settings; each evaluator is its own run.", "First run: OpenBMB UltraEval-Audio replication documents (9 models)." ], "why_new": "No CV3 zero-shot Chinese id exists (only cv3-eval-hard-zh-cer and the 9-language macro). UltraEval-Audio prints it for 9 catalogue models in one evaluator run; the en / hard-en / hard-zh cells of the same tables use existing ids. suggested_tasks empty to match the other cv3-eval-* ids, which are not in voice-cloning." }, { "id": "minimax-ml21-ultraeval-macro-err", "name": "MiniMax multilingual test set · 21-language mean error (UltraEval-Audio common set)", "category": "tts", "kind": "err", "url": "https://huggingface.co/datasets/MiniMaxAI/TTS-Multilingual-Test-Set", "metric": "mixed_word_character_error_rate", "setup": "MiniMax-Speech multilingual zero-shot voice-cloning test set (MiniMaxAI/TTS-Multilingual-Test-Set, 100 items per language), scored with the UltraEval-Audio minimax evaluator: Whisper large-v3 for every language except Chinese (Seaco Paraformer large); CER for Chinese, Cantonese, Japanese, Korean and Thai, WER otherwise (incl. Arabic, Hindi, Greek, Vietnamese). Unweighted mean over 21 languages: Arabic, Cantonese, Chinese, Czech, Dutch, English, Finnish, French, German, Greek, Hindi, Japanese, Korean, Polish, Portuguese, Romanian, Russian, Spanish, Thai, Turkish, Vietnamese. Per-language cells are provenance, not extra results.", "score_unit": "error ratio (mean of source % / 100)", "caveats": [ "Different language set and scorer from minimax-ml23-macro-err (Boson AI, 23 languages) and voxtral-tts-minimax9-wer (Voxtral ASR); do not pool.", "Cantonese CER (35-46%) dominates the mean for every system.", "Equal-weight derived mean of printed cells." ], "why_new": "UltraEval-Audio prints per-language MiniMax tables for four catalogue models (fish-audio-s2-pro, fireredtts3-base, moss-tts-v1-5, voxcpm2) with one scorer stack; no existing id has this language set or scorer. suggested_tasks voice-cloning by analogy with voxtral-tts-minimax9-wer (same test set, zero-shot cloning)." }, { "id": "livebench-20240831", "name": "LiveBench 2024-08-31", "category": "llm", "kind": "acc", "url": "https://github.com/LiveBench/LiveBench/blob/24364d65076429adfcba7be18af4d44fddc43dce/changelog.md", "description": "LiveBench release 2024-08-31 overall average (0–1): the 1,000 public questions live at that release (math 232, reasoning 150, coding 128, language 140, data analysis 150, instruction following 200; the 2024-08-31 update refreshed AMPS_Hard, olympiad (IMO/USAMO 2024) and math_comp (reordered AMC 2023) questions), objective ground-truth scoring, mean of the six category means where each category is the mean of its task means (LiveBench show_livebench_result.py). Not merged with livebench-20240726 or livebench-20241125 (different question sets). Recorded only for models with no 2024-07-26 result (one LiveBench release per model, so overlapping releases don't count twice)." }, { "id": "freya-tr-eval-wer", "name": "Freya-TR-Eval Turkish TTS intelligibility (WER, faster-whisper large-v3, 8 kHz band)", "category": "tts", "kind": "err", "url": "https://huggingface.co/datasets/freyavoice/freya-tr-eval", "metric": "word_error_rate", "setup": "Freya-TR-Eval (freyavoice/freya-tr-eval, CC BY 4.0): 495 raw Turkish sentences (everyday text with numbers, dates, money, abbreviations). Each system speaks every sentence (3 seeds or takes, default/fixed voice); clips are band-limited to 8 kHz and back to 16 kHz, transcribed by Whisper large-v3 (faster-whisper, beam 5, language tr), reference and hypothesis get Freya's Turkish normalisation, and corpus WER is computed over the whole set with jiwer; the score is the mean over the 3 seeds, as printed. Two evaluator runs: the EMA Lightning card ('Our runs', RTX 4090) and Patientdesk.ai's independent re-run (EMA Lightning's published eval.py, NVIDIA L4) printed on the Antalia-2 Mini card.", "score_unit": "error ratio (printed WER % / 100)", "caveats": [ "Single-language (Turkish) intelligibility only; says nothing about naturalness or voice cloning.", "Automatic ASR-judged metric with one ASR model; both evaluators also build Turkish TTS models (their own rows are by: lab).", "Cloud APIs run with default voices (ElevenLabs 'George', Gemini 'Kore'); cloning systems use the Anka male reference clip.", "Coqui GlowTTS input is lowercased the Turkish way first (the front end that model needs)." ] }, { "id": "multitextedit-semavg", "name": "MultiTextEdit cross-lingual text-in-image editing (SemAvg, LVM judge)", "category": "image", "kind": "acc", "url": "https://arxiv.org/html/2605.08163v2", "metric": "semantic_average_0_5", "setup": "MultiTextEdit (Harbin Institute of Technology, arXiv:2605.08163v2, CC BY 4.0): 3,600 text-in-image editing instances across 12 typologically diverse languages/scripts (insertion, replacement, deletion of rendered text), each with a human-annotated reference image. An LVM judge scores instruction following, text accuracy, visual consistency, layout preservation and language/script fidelity on 0-5 (two-stage protocol for script fidelity; human agreement QWK 0.76 on 240 samples). SemAvg = mean of the five dimensions over all instances, as printed in Table 3; stored as SemAvg/5. The authors ran every system themselves with the provider's official pipeline and highest-quality recommended settings (open models on 8xA800; proprietary via official APIs).", "score_unit": "fraction of the 0-5 judge scale (printed SemAvg / 5)", "caveats": [ "Judge model not named in the text; scores are LVM-judged, validated against one native speaker per language.", "Thinking-enabled variants are listed as separate settings; for BAGEL the thinking row is used (highest reasoning setting), as on Edit-Compass.", "Step1X-Edit rows not imported: the paper does not say which Step1X-Edit version (the thinking variant implies v1.2)." ] }, { "id": "whatif-world-apeo-pavg", "name": "What-If World causal-intervention video generation (APEO paired average, Gemini 3.1 Pro judge)", "category": "video", "kind": "acc", "url": "https://arxiv.org/html/2605.27589v1", "metric": "pass_rate", "setup": "What-If World (UCLA / University of Tennessee, arXiv:2605.27589v1, CC BY 4.0): 319 driving (nuScenes) and robot-manipulation (DROID) scenes; from one real initial frame each model generates two videos for a minimally different prompt pair (an intervention on one causal variable: force, timing, friction, material, obstacle, alignment). Gemini 3.1 Pro judges Adherence, Physics, Environment and Outcome on the pair; pAvg (the paper's primary paired metric) is the mean paired pass rate, as printed in Table 2; stored /100. Open models run by the authors with their public default inference settings on RTX A6000s; closed models via hosted APIs (kie.ai).", "score_unit": "fraction (printed pass rate % / 100)", "caveats": [ "Closed models accessed through the kie.ai reseller; vendor tiers not stated.", "Not imported: CogVideoX1.5-5B (T2V vs I2V checkpoint not stated), 'Cosmos-Predict2-2B' (the table names Predict2 but the citation is the Cosmos-Predict2.5 report), 'Kling 3.0' (cited as the Kling-Omni report, so Kling 3.0 vs Kling 3.0 Omni is unclear)." ] }, { "id": "cronos-i2v-success-rate", "name": "CRONOS counterfactual rigid-body physics (I2V success rate)", "category": "video", "kind": "acc", "url": "https://arxiv.org/html/2605.23699v1", "metric": "success_rate", "setup": "CRONOS (University of Freiburg / MPI Informatics / CISPA, arXiv:2605.23699v1, CC BY 4.0): synthetic rigid-body scenes (falls, collisions) with controlled counterfactual interventions; each model continues the scene from the first frame (I2V). Per-video metrics (background stability, motion similarity to the simulator ground truth, appearance and 3D-shape stability, physical plausibility) are thresholded with human-calibrated cut-offs; a video counts as a success only if every metric passes. Score = fraction of successful videos averaged over all videos, as printed in Table 1 (I2V block). All models run by the authors with open weights.", "score_unit": "fraction (printed success rate)", "caveats": [ "Success rates are near the floor for weaker models (0.01-0.02), so the test separates mainly the top models.", "Only the I2V block is used; the V2V block (five conditioning frames, Cosmos and MAGI-1 only) is a different setting.", "CogVideoX1.5-5B row not imported (the catalogue has no CogVideoX1.5 I2V checkpoint entry)." ] }, { "id": "agentworldbench-overall", "name": "AgentWorldBench (open-ended, overall)", "kind": "acc", "category": "llm", "url": "https://github.com/QwenLM/Qwen-AgentWorld", "description": "Qwen-AgentWorld's environment-simulation benchmark: a model predicts the next environment state for agent actions in seven domains (MCP, Search, Terminal, SWE, Android, Web, OS); five-dimensional rubric mean per domain on a 0-100 scale, Overall column. Measures world-model fidelity, not task success." }, { "id": "livecodebench-v2", "name": "LiveCodeBench v2", "kind": "acc", "category": "llm", "url": "https://github.com/LiveCodeBench/LiveCodeBench", "description": "LiveCodeBench release_v2 code-generation pass@1 (problems published May 2023 to May 2024, 511 problems; 'LCBv2' / 'LCBv2 All' in Open Thoughts' Evalchemy tables); distinct from the v3-v6 windows and from per-difficulty subsets." }, { "id": "puredocbench-avg3", "name": "PureDocBench document parsing (Avg3 over Clean/Digital/Real)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2605.07492v2", "description": "PureDocBench (arXiv:2605.07492v2, CC BY 4.0): full-page document parsing of 1,475 generated source pages in 10 domains, each as matched Clean, Digital-degraded and Real (recaptured) images (4,425 images). OmniDocBench framework: per-track Overall = ((1-TextEdit)x100 + FormulaCDM + TableTEDS)/3; score = mean of the three track Overalls (Avg3), Table 2 'Overall', stored /100.", "chance": 0 }, { "id": "swe-atlas-mini-swe-agent", "name": "SWE Atlas (Codebase Q&A, Test Writing, Refactoring; mini-SWE-agent) pass@1", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2605.08366v1", "description": "SWE Atlas (arXiv:2605.08366v1, CC BY 4.0): 284 expert-authored software-engineering tasks from 18 open-source repositories - Codebase Q&A (124), Test Writing (90) and Refactoring (70) - verified by programmatic checks (tests, mutation testing) plus rubric-based LLM-judge grading. Score = Pass@1 averaged over 3 trials with the minimal mini-SWE-agent (bash-only) scaffold in Harbor, highest thinking setting per Table 8 (Table 1).", "chance": 0 }, { "id": "diagnosticiq-macro", "name": "DiagnosticIQ industrial maintenance MCQ (macro accuracy, 4 options)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2605.08614v1", "description": "DiagnosticIQ (arXiv:2605.08614v1, CC BY 4.0): 6,690 four-option multiple-choice questions generated from 118 expert-curated industrial monitoring rules (HVAC/AHU, pumps, chillers ...): given an asset description and the rule's triggered sensor conditions, select (or eliminate) the correct maintenance observation/action. Zero-shot direct prompting, temperature 0, max 4,096 tokens; missing answers count as wrong. Score = accuracy macro-averaged over asset classes (Table 2 'Macro % D.IQ'); chance 25%.", "chance": 0.25 }, { "id": "contractbench-sr", "name": "ContractBench success rate (33 observation-contract tool tasks)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2605.17281v2", "description": "ContractBench (arXiv:2605.17281v2, CC BY 4.0): 33 programmatically validated tool-use agent tasks that test whether an agent preserves 'observation contracts' (temporal validity of presigned URLs/tokens under a virtual clock, and byte-level integrity checked by SHA-256). Success rate = passed episodes / 99 (33 tasks x k=3 rollouts, temperature 0, 600 s per episode), as printed in Table 10 (SR%).", "chance": 0 }, { "id": "chartarena-en-map", "name": "ChartArena chart parsing (English, mAP_high average)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2606.01348v3", "description": "ChartArena (arXiv:2606.01348v3, CC BY 4.0): chart-to-structure parsing across eight chart families (bar, line, pie, radar, box plot, combination, flowchart, mind map) and three visual scenarios (digital rendering, printed photo, hand-drawn photo). Outputs in any format are normalised to triples (numeric charts) or labelled graphs (diagrams) and scored by mAP at the 'high' tolerance; score = English 'Average' column of Table 2 / Table C.3, stored /100.", "chance": 0 }, { "id": "k-browsecomp-verified", "name": "K-BrowseComp-Verified (Korean web-browsing agent questions, pass@1)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2606.02404v1", "description": "K-BrowseComp (arXiv:2606.02404v1, CC BY 4.0): 300 human-verified BrowseComp-style questions grounded in Korean contexts (multi-hop or parallel-constraint, >=4 steps). Agents run in Perplexity's search_evals deep-research agent with Perplexity Search and a budget of 10 search calls per question; final answers extracted by GPT-5.4-mini and matched to gold. Score = single-run accuracy (pass@1) on the Verified subset (Table 2).", "chance": 0 }, { "id": "kina-accuracy", "name": "KINA (Knowledge Index of Noah's Ark) accuracy, 899 ten-option items", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2606.05104v2", "description": "KINA (arXiv:2606.05104v2, CC BY 4.0): 899 expert-written pseudo-multiple-choice knowledge items across 12 disciplines / 261 subfields; each item has 10 combinatorial options over several statements (chance 10%). Score = overall avg@4 accuracy, default temperature, 32K max new tokens, reasoning budget 16K or 'Medium', no tools (Table 3 'ALL').", "chance": 0.1 }, { "id": "surgiq-zeroshot-logprob", "name": "SurgiQ surgical MCQ (zero-shot, log-probability scoring)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2606.08071v1", "description": "SurgiQ (arXiv:2606.08071v1, CC BY 4.0): ~13k four-option multiple-choice questions generated from surgical textbooks across six specialties (physician-audited, near-duplicate filtered). Zero-shot fixed prompt; the answer is the option letter with the highest next-token log-probability (no generated reasoning, so reasoning models are scored without thinking). Score = micro-accuracy over all items (Table 4 'Avg.'); chance 25%.", "chance": 0.25 }, { "id": "t1-bench-pass-hat-3", "name": "T1-Bench multi-domain conversational agent pass^3", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2606.11070v1", "description": "T1-Bench (arXiv:2606.11070v1, CC BY 4.0): persistent multi-session tool-using customer-service conversations across 11 travel/dining/shopping domains and multi-domain combinations, with a Gemma4-26B-A4B-it user simulator. A conversation passes only if tool-call recall = 1, parameter recall = 1 and all tool outputs match exactly; score = Pass^K with K=3 (all three of three temperature-1.0 trials pass), averaged over domains (Table 2).", "chance": 0 }, { "id": "polish-medical-exams-om", "name": "Polish medical board exams, bias-mitigated MCQ protocol (7-exam average)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2606.12250v1", "description": "Polish Medical Exams re-evaluation (arXiv:2606.12250v1, CC BY-SA 4.0): five-option (A-E) multiple-choice questions from Polish medical and dental final exams (LEK, LDEK), specialist board exams (PES, PESDL, PESF) and English versions of LEK/LDEK. 'OM' protocol: neutral instruction without a sample answer letter, constrained generation to the option letters, temperature 0. Score = mean accuracy over the seven exams (Table 4 'Avg OM'); chance 20%.", "chance": 0.2 }, { "id": "viseditbench-pass", "name": "VisEditBench strict pass rate (1,395 visualization-code editing tasks)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2608.10408v1", "description": "VisEditBench (arXiv:2608.10408v1, CC BY 4.0): 1,395 human-annotated tasks where a VLM receives visualization code (Matplotlib/Vega-Lite), a rendered buggy, human-marked or reference chart and an instruction, and must return revised executable code. Zero-shot, temperature 0.2, max 4,096 output tokens. Rendered output passes only if executable, task accuracy >=4.5/5, readability and visual quality >=4/5 and visual similarity >=90 (GPT-4o rubric judge, validated against human ratings). Score = Overall pass rate (Table 2).", "chance": 0 }, { "id": "diagram-mmu-dqa", "name": "Diagram-MMU diagram question answering (DQA accuracy, scientific TikZ diagrams)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2608.12262v1", "description": "Diagram-MMU (arXiv:2608.12262v1, CC BY 4.0): diagram question answering over 3.7k curated scientific TikZ diagrams in six domains (graph structures, charts, planar geometry, circuits, 3D shapes, chemistry); descriptive, standard-reasoning and what-if questions, two per diagram. Answers extracted and graded 0/1 by Qwen3-Next-80B-A3B-Instruct (97.5% agreement with humans). Direct answering, temperature 1.0. Score = DQA accuracy (Table 5).", "chance": 0 }, { "id": "bear-bench-overall", "name": "BEAR-Bench (bilingual RU/EN enterprise and academic document images, overall accuracy)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2608.17895v1", "description": "BEAR-Bench (arXiv:2608.17895v1, CC BY 4.0): 1,000 human-authored question-answer pairs over text-dense professional and academic document images (business 532, science 468; Russian 618, English 382; figures, diagrams, equations). Zero-shot, image + raw question, temperature 0.6; free-form answers judged correct/incorrect by GPT-4o for semantic equivalence. Score = Overall accuracy (Table 2).", "chance": 0 }, { "id": "thinkingbox-bench-pass1", "name": "Thinkingbox-bench pass@1 (507 stateful business-workflow agent tasks, N=20)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2608.19741v4", "description": "Thinkingbox-bench (arXiv:2608.19741v4, CC BY 4.0): 507 tool-using agent tasks in five stateful business domains (retail, auto insurance, booking, neobank support, consulting) with MCP-compatible tools, a GPT-5.4-mini simulated user and executable backend-state checks (plus a response rubric on some tasks). A trial passes only if every executable check passes; score = pass@1 micro-averaged over tasks and N=20 independent trials (Table 15 / Table 4 'Average'). Agent temperature 1.0, reasoning effort medium.", "chance": 0 }, { "id": "scidocbench-overall", "name": "SciDocBench scientific-document workflows (overall, 496 instances)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2609.05141v2", "description": "SciDocBench (arXiv:2609.05141v2, CC BY 4.0): 124 expert-authored, difficulty-screened scientific-research tasks over full papers, multiple papers and associated code/data (seven capability groups, 19 subtasks), each asked in English and Chinese with two document representations (all page images first, or Markdown interleaved) = 496 instances. Scored by deterministic rules, a rubric LLM judge or execution; failures score 0. Score = Overall mean (Table 2), stored /100.", "chance": 0 }, { "id": "gamelogicbench-claude-code", "name": "GameLogicBench solve rate (72 Godot game-logic tasks, Claude Code scaffold)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2609.21562v2", "description": "GameLogicBench (arXiv:2609.21562v2, CC BY 4.0): 72 coding-agent tasks (21 Atom, 28 Combo, 23 Repo) where the agent implements runtime game logic in a Godot 4.4 project; solutions are judged by deterministic tick-level state assertions over hidden scenarios. Score = average solve rate over all tasks (Table 3 'Average'), one sample per configuration, effort=high, 3,600 s limit, egress sealed, Claude Code 2.1.177 scaffold for every imported row.", "chance": 0 }, { "id": "eegagentbench-overall", "name": "EEGAgentBench overall (tool-based EEG analysis agents, 6 tasks)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2609.31632v1", "description": "EEGAgentBench (arXiv:2609.31632v1, CC BY 4.0): 1,072 instances from six public EEG datasets over six tasks - EEG knowledge QA (T1, direct answer) and five tool-based analyses (artifact detection, abnormality screening, dementia cohort classification, seizure event detection, full-night sleep staging) where the agent calls 10 deterministic signal-processing tools before answering. Single run per instance. Score = Overall, the unweighted mean of T1 accuracy, Macro-F1 (T2-T4, T6) and event-based F1 (T5), on a 0-1 scale (Table 2).", "chance": 0 }, { "id": "e2e-swe-pass1", "name": "E2E-SWE (build a working codebase from scratch, 186 tasks) pass@1", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2609.38335v1", "description": "E2E-SWE (arXiv:2609.38335v1, CC BY 4.0): 186 tasks where an agent must build a complete working repository from a natural-language specification; a task counts only if the generated repository passes every hidden test. Simple ReAct scaffold (bash + edit + submit tools), 4-hour budget, strongest reliable reasoning effort per model; score = pass@1 averaged over 4 independent runs (Table 2).", "chance": 0 }, { "id": "klinikebench-strict-pass1", "name": "KlinikeBench strict pass@1 (333 interactive clinical encounters, Terminus-2)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2609.38480v2", "description": "KlinikeBench (arXiv:2609.38480v2, CC BY 4.0): 333 clinician-authored interactive clinical encounters run in Harbor with the Terminus-2 ReAct harness; the model questions an LLM-simulated patient (GPT-5.4-mini) and executes sandboxed clinical tools. Strict pass@1 = share of tasks with correct diagnosis AND all required tools AND argument-level action constraints AND full must-ask coverage; one rollout per task, medium reasoning effort, 32-turn cap (Table 7).", "chance": 0 }, { "id": "longsocialbench-full", "name": "LongSocialBench (full-context, 4-option MCQ over discussion threads)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2610.04118v1", "description": "LongSocialBench (arXiv:2610.04118v1, CC BY 4.0): 1,462 human-authored, independently verified four-option questions over complete long online discussion threads (Hacker News, Stack Exchange, Reddit) testing reply-structure-aware social reasoning (six capability surfaces). Full-context prompting: whole serialized thread with reply links + stem + options, one answer letter; exact-match micro accuracy over parsed responses (Table 13 'Full'). Chance 25%.", "chance": 0.25 }, { "id": "video2world-v2wscore-opencode", "name": "Video2World V2WScore (coding agents reconstruct interactive simulated worlds from videos, OpenCode scaffold)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2610.04432v2", "description": "Video2World (arXiv:2610.04432v2, CC BY-SA 4.0): 222 instances (189 embodied manipulation video clips, 39 task families); a coding agent must build an executable physics-simulated scene and robot behaviour that reproduce the video. V2WScore (0-100) combines build rate, task success/progress, geometric fidelity and dynamic fidelity, macro-averaged over task families (Table 2), stored /100. One autonomous attempt per instance, 180-minute limit, OpenCode 1.18.29 scaffold for every imported row (other scaffolds excluded).", "chance": 0 }, { "id": "spacecast-bench-overall", "name": "SpaceCast-Bench predictive spatial reasoning (overall, 16 task types)", "kind": "acc", "category": "llm", "url": "https://arxiv.org/html/2610.12402v1", "description": "SpaceCast-Bench (arXiv:2610.12402v1, CC BY 4.0): 3,862 multiple-choice questions from 182 ScanNet/ScanNet++ scenes over 16 task types; the model sees depth-verified view chains and must predict spatial outcomes (direction, distance, occlusion, attributes) after hypothetical camera/object/world transformations at three levels (static, local, global). Chain-of-thought prompt, deterministic decoding. Score = unweighted macro-average of per-task accuracy (Table 1 'Overall'); random baseline 25.9%, humans 87.2%.", "chance": 0.259 }, { "id": "legal-research-bench-allpass", "name": "Legal Research Bench (agentic, all-pass)", "kind": "acc", "category": "llm", "chance": 0, "url": "https://arxiv.org/abs/2610.00609v1", "description": "Legal Research Bench (Vals AI, arXiv:2610.00609v1, CC BY 4.0): 413 original U.S. legal research questions written by practising attorneys across eight areas of law. Models run as tool-using agents (web search, case-law search, page parsing, retrieval). Score = all-pass: an answer counts only if it meets every attorney-written rubric criterion and every cited authority verifies. Graded by a GPT-5.4 judge validated against attorneys." }, { "id": "legal-research-bench-no-tools-allpass", "name": "Legal Research Bench (single-shot, no tools, all-pass)", "kind": "acc", "category": "llm", "chance": 0, "url": "https://arxiv.org/abs/2610.00609v1", "description": "Legal Research Bench (Vals AI, arXiv:2610.00609v1, CC BY 4.0) tool ablation, Table 4: the same 413 attorney-written U.S. legal research questions answered in one shot with no tool access. Score = all-pass (every rubric criterion met and every cited authority valid), GPT-5.4 judge." }, { "id": "finance-agent-benchmark-v1-class-balanced", "name": "Finance Agent Benchmark v1 (537 questions, class-balanced accuracy)", "kind": "acc", "category": "llm", "chance": 0, "url": "https://arxiv.org/abs/2508.00828v1", "description": "Finance Agent Benchmark as published in the paper (Vals AI, arXiv:2508.00828v1, CC BY 4.0): 537 expert-written financial-analyst research questions over recent SEC filings in nine task categories, answered by a tool-using agent (Google web search via SerpAPI, EDGAR full-text search, HTML page parsing, retrieval over stored pages). Rubric grading by an LLM judge. Score = class-balanced accuracy: accuracy per task category, averaged with equal weight. Not the later Finance Agent v1.1 or v2 leaderboards on vals.ai." }, { "id": "vibe-code-bench-test", "name": "Vibe Code Bench (test split, 50 web apps)", "kind": "acc", "category": "llm", "chance": 0, "url": "https://arxiv.org/abs/2603.04601v3", "description": "Vibe Code Bench as published in the paper (Vals AI, arXiv:2603.04601v3, CC BY 4.0): build a complete, deployable web application (React + Vite, Supabase, Docker Compose; email and Stripe integrations where specified) from a natural-language specification inside an OpenHands-based sandbox with terminal, browser and web search, within 5 hours of wall-clock time. A Browser Use agent (Claude Sonnet 4.5) runs the app's pre-written user workflows; a workflow passes when at least 90% of its substeps succeed. Score = share of workflows passed per app, averaged over the 50 held-out test apps (473 workflows)." }, { "id": "formalproofbench", "name": "FormalProofBench (200 graduate-level Lean 4 problems, agentic)", "kind": "acc", "category": "llm", "chance": 0, "url": "https://arxiv.org/abs/2603.26996v1", "description": "FormalProofBench (Vals AI, arXiv:2603.26996v1, CC BY 4.0; Vals' 'ProofBench'): 200 private advanced-undergraduate and graduate problems from qualifying exams and textbooks (analysis, probability, functional analysis, number theory, logic, algebra, algebraic geometry), each given as a natural-language statement plus a Lean 4 formal statement. The model works as an agent with Mathlib search (Loogle), Lean code execution and a submit tool, up to 40 turns; a problem counts only if the submitted proof passes the Lean 4 kernel (Lean v4.25.2, Mathlib c98ae54; axiom and local_instance rejected). Score = share of the 200 problems solved, as printed in Table 2." }, { "id": "bfcl-v4-2025-11-03", "name": "BFCL V4 (2025-11-03 archive, models absent from the December archive)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-11-03/score/data_overall.csv", "description": "Berkeley Function Calling Leaderboard V4 Overall Acc from the 2025-11-03 result archive, recorded only for models that are absent from the 2025-12-16 archive (bfcl-v4). Kept apart from bfcl-v4 because the archive re-scores entries between snapshots. FC entry when available, otherwise Prompt." }, { "id": "bfcl-v4-2025-08-26", "name": "BFCL V4 (2025-08-26 archive, models absent from later archives)", "kind": "acc", "chance": 0, "category": "llm", "url": "https://github.com/HuanzhiMao/BFCL-Result/blob/cadd76a8dc62984e7bf15ecaf358f070bae745cc/2025-08-26/score/data_overall.csv", "description": "Berkeley Function Calling Leaderboard V4 Overall Acc from the 2025-08-26 result archive, recorded only for models absent from the 2025-11-03 and 2025-12-16 archives. Kept apart from bfcl-v4 and bfcl-v4-2025-11-03 because the archive re-scores entries between snapshots. FC entry when available, otherwise Prompt." }, { "id": "nsanku-ghana7-macro-cer", "name": "nsanku TTS Benchmark · 7 Ghanaian languages mean CER (GhanaNLP)", "category": "tts", "kind": "err", "url": "https://huggingface.co/spaces/ghananlpcommunity/nsanku-tts-benchmark/blob/41b828ed6803bd052b1bef94bb6802fecd2edf8c/bundled_data.json", "metric": "character_error_rate", "setup": "nsanku TTS Benchmark (ghananlpcommunity/nsanku-tts-benchmark): 200 sentences per language in Dangme, Dagbani, Dagaare, Fante, Ga, Gonja and Akuapem Twi (Bible, JW, UNICEF, WAXAL, LDS, finance domains); each clip transcribed by a per-language ASR judge (KhayaAI/khaya-asr-v3 for five languages, Qlerqly/griot-nano-1 for Ga and Akuapem Twi) and scored by CER against the sentence. Unweighted mean of the 7 per-language CERs; per-language cells are provenance, not extra results. One entry per model: reference-audio (ref) mode where the model has one.", "score_unit": "error ratio", "caveats": [ "The benchmark makers (GhanaNLP) also make ghana-tts-72k and the Khaya ASR judge; their own model's row is by: lab.", "Judge CER on real human speech is itself 4-26% per language (printed 'cer_on_real_speech'), so absolute values are high.", "Equal-weight derived mean of printed per-language cells." ], "why_new": "No existing id covers Ghanaian-language TTS; the Space (MIT) prints all 7 languages for three catalogue models (gemini-3-1-flash-tts-preview, omnivoice, ghana-tts-72k)." }, { "id": "livecodebench-official-20231101-20250201", "name": "LiveCodeBench official release_v5 (Nov 1, 2023–Feb 1, 2025)", "kind": "acc", "category": "llm", "url": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json", "family": "livecodebench-codegeneration", "description": "Official website code-generation pass@1 from the release_v5 data file (build/v5.json). Inclusive UTC filter 2023-11-01 00:00:00 through 2025-02-01 00:00:00 (the v5 page's default slider bounds); 629 identical problem IDs per selected model. Mean then one-decimal percentage rounding as upstream. No difficulty components or repaired/composite scores. Only models with no other LiveCodeBench window are recorded here.", "window_start": "2023-11-01T00:00:00Z", "window_end": "2025-02-01T00:00:00Z", "window_end_inclusive": true, "problem_count": 629, "release_commit": "f186ab5041d6b768733b8ae8145151c867942134" }, { "id": "polyrating-chatbot-arena-rating", "name": "Polyrating rating (bias-aware re-fit of Chatbot Arena votes, mid-2024)", "category": "llm", "kind": "elo", "url": "https://arxiv.org/html/2409.00696v3#A6.T7", "description": "Polyrating main rating (Elo scale) from Table 7 of arXiv 2409.00696v3: the authors refit the public Chatbot Arena pairwise human votes (114 models, up to mid-2024) with a multidimensional Bradley-Terry model that separates response quality from task/prompt-type modifiers (English, Chinese, hardness, code). Same votes as lmarena-text, different rating model and snapshot; not merged with lmarena-text." }, { "id": "charxiv-rq-test", "name": "CharXiv reasoning questions (test split)", "category": "llm", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2406.18521v1#A3.T4", "description": "CharXiv reasoning-question accuracy on the held-out TEST split (1,323 charts, answers kept private, scored by the CharXiv authors), as printed in Appendix C Table 4 of arXiv:2406.18521v1. Different questions from charxiv-rq (validation split); not merged with it." }, { "id": "charxiv-dq-test", "name": "CharXiv descriptive questions (test split)", "category": "llm", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2406.18521v1#A3.T4", "description": "CharXiv descriptive-question accuracy on the held-out TEST split (1,323 charts, answers kept private, scored by the CharXiv authors), as printed in Appendix C Table 4 of arXiv:2406.18521v1. Different questions from charxiv-dq (validation split); not merged with it." }, { "id": "chineseword-longcat-overall", "name": "ChineseWord (LongCat-Image 8,105-character set, PPOCRv5) overall", "category": "image", "kind": "acc", "chance": 0, "url": "https://arxiv.org/html/2512.07584v1#S5.T7", "description": "Chinese character rendering accuracy on the 8,105-prompt ChineseWord set built by the LongCat-Image authors (one prompt per character of the General Standard Chinese Characters Table, standardized template, following Qwen-Image's protocol), scored with PPOCRv5 OCR; Overall accuracy over levels L1-L3, divided by 100. Not Qwen-Image's own ChineseWord numbers." }, { "id": "aura-ar-en-22set-avg-wer", "name": "QCRI Arabic-English ASR leaderboard, 22-set average WER", "category": "stt", "kind": "err", "url": "https://huggingface.co/spaces/QCRI/AURA-Arabic-English-ASR-leaderboard/blob/208ff13439f377131207a2b32bf73364ba0e2d0c/leaderboard.tsv", "description": "Average WER (fraction) over the 22 Arabic, English and code-switched test sets of QCRI's bilingual ASR leaderboard (arzen, Casablanca, Common Voice ar (+AURA split), DACS, ESCWA code-switch, EveryAyah, FLEURS ar/code-switch/en, L2-ARCTIC, LibriSpeech test-clean/other, MASC clean/noisy, MGB-2/3/5, Mixat, QASR code-switch, RDI, SADA), scored by QCRI with one bilingual normalizer (diacritics removed, Alef variants unified, punctuation removed, Latin lower-cased). As printed in the 'Average WER' column." }, { "id": "ahelm-voxceleb2-em", "name": "AHELM VoxCeleb2 same-speaker detection (exact match)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2508.21376v2", "description": "Audio-LLM task from AHELM (Stanford CRFM, HELM framework): given two VoxCeleb2 audio clips, answer whether the speakers are the same; exact-match accuracy, temperature 0. Not an ASR test. Source: AHELM, arXiv 2508.21376v2 (CC BY 4.0).", "chance": 0.5 }, { "id": "ahelm-vocalsound-pem", "name": "AHELM VocalSound human-sound recognition (prefix exact match)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2508.21376v2", "description": "AHELM scenario: recognise the human vocal sound in a VocalSound clip (laughter, sigh, cough, throat clearing, sneeze, sniff); pseudo/prefix exact match, temperature 0. Source: AHELM, arXiv 2508.21376v2 (CC BY 4.0)." }, { "id": "ahelm-librispeech-wer", "name": "AHELM LibriSpeech transcription by audio LLMs (WER)", "category": "llm", "kind": "err", "url": "https://arxiv.org/html/2508.21376v2", "description": "AHELM scenario: an audio-language model is prompted to transcribe LibriSpeech audio; word error rate (fraction, lower is better), temperature 0, max 200 output tokens. For audio LLMs (category llm); dedicated speech recognisers are scored on the stt benchmarks instead. Source: AHELM, arXiv 2508.21376v2 (CC BY 4.0)." }, { "id": "ahelm-airbench-foundation-music-em", "name": "AHELM AIR-Bench Foundation music subsets (exact match)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2508.21376v2", "description": "AHELM knowledge scenario: AIR-Bench (Foundation) single-choice questions, music subsets only (music genre recognition, instrument classification, music QA); exact match, temperature 0. Source: AHELM, arXiv 2508.21376v2 (CC BY 4.0)." }, { "id": "ahelm-corebench-pem", "name": "AHELM CoRe-Bench conversational audio reasoning (pseudo-exact match)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2508.21376v2", "description": "CoRe-Bench, introduced in the AHELM paper: 2,290 questions over 2,082 synthetic multi-turn audio conversations whose answers must be inferred from the dialogue; pseudo-exact match, temperature 0. Source: AHELM, arXiv 2508.21376v2 (CC BY 4.0).", "chance": 0 }, { "id": "ahelm-meld-audio-pem", "name": "AHELM MELD audio emotion classification (pseudo-exact match)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2508.21376v2", "description": "AHELM emotion scenario: classify one of seven emotions (anger, disgust, sadness, joy, neutral, surprise, fear) from MELD (Friends) audio clips only; pseudo-exact match, temperature 0. Source: AHELM, arXiv 2508.21376v2 (CC BY 4.0)." }, { "id": "ahelm-mustard-em", "name": "AHELM MUStARD audio sarcasm detection (exact match)", "category": "llm", "kind": "acc", "url": "https://arxiv.org/html/2508.21376v2", "description": "AHELM emotion scenario: detect sarcasm from the audio of MUStARD sitcom utterances (with context); binary exact match, temperature 0. Source: AHELM, arXiv 2508.21376v2 (CC BY 4.0).", "chance": 0.5 }, { "id": "ahelm-fleurs-7lang-wer", "name": "AHELM FLEURS 7-language transcription by audio LLMs (WER)", "category": "llm", "kind": "err", "url": "https://arxiv.org/html/2508.21376v2", "description": "AHELM multilinguality scenario: transcribe FLEURS audio in Finnish, Mandarin Chinese, Thai, Hebrew, Bengali, English and Zulu; mean WER (fraction; can exceed 1 when models produce long hallucinated output), temperature 0, max 200 output tokens. Source: AHELM, arXiv 2508.21376v2 (CC BY 4.0)." }, { "id": "ahelm-mls-7lang-wer", "name": "AHELM Multilingual LibriSpeech 7-language transcription by audio LLMs (WER)", "category": "llm", "kind": "err", "url": "https://arxiv.org/html/2508.21376v2", "description": "AHELM multilinguality scenario: transcribe Multilingual LibriSpeech audio in Italian, French, Polish, Dutch, Portuguese, Spanish and German; mean WER (fraction), temperature 0. Source: AHELM, arXiv 2508.21376v2 (CC BY 4.0)." }, { "id": "ahelm-srb-librispeech-noise-wer", "name": "AHELM Speech Robust Bench (LibriSpeech-clean with noise) transcription WER", "category": "llm", "kind": "err", "url": "https://arxiv.org/html/2508.21376v2", "description": "AHELM robustness scenario: transcribe LibriSpeech-clean audio from Speech Robust Bench with Gaussian or environmental noise at levels 1-3 (four subsets); mean WER (fraction), temperature 0. Source: AHELM, arXiv 2508.21376v2 (CC BY 4.0)." }, { "id": "halueval-qa-acc", "name": "HaluEval QA hallucination detection (accuracy)", "category": "llm", "kind": "acc", "chance": 0.5, "url": "https://github.com/RUCAIBox/HaluEval", "description": "HaluEval QA split as run in the Hallucinations Leaderboard (lm-eval task 'halueval_qa'): given a knowledge passage, question and answer, decide whether the answer contains a hallucination (yes/no); accuracy over 10,000 samples." }, { "id": "medqa-4options", "name": "MedQA (USMLE, 4 options)", "category": "llm", "kind": "acc", "chance": 0.25, "url": "https://github.com/jind11/MedQA", "description": "MedQA USMLE-style medical exam questions, English 4-option test set (1,273 questions), zero-shot accuracy by likelihood ranking (lm-eval task 'medqa_4options')." }, { "id": "medmcqa", "name": "MedMCQA", "category": "llm", "kind": "acc", "chance": 0.25, "url": "https://medmcqa.github.io/", "description": "MedMCQA Indian medical entrance exam questions (4 options), validation split (4,183 questions), zero-shot accuracy by likelihood ranking (lm-eval task 'medmcqa')." } ]